From f7b0f074cf269ee69b9ee334cc834d6a82eaa740 Mon Sep 17 00:00:00 2001 From: Martyn Ranyard Date: Sat, 5 Sep 2020 18:14:56 +0200 Subject: [PATCH] Redis refactor and video download script. Signed-off-by: Martyn Ranyard --- .../src/Components/AppAdmin/AppAdmin.js | 28 ++- .../templates/deployment-db.yaml | 66 +++++ .../templates/deployment.yaml | 2 + .../twitchsingstools/templates/pvc-db.yaml | 12 + .../templates/service-db.yaml | 15 ++ .../helm/twitchsingstools/values-dev.yaml | 2 +- deployments/helm/twitchsingstools/values.yaml | 9 + internal/data/data.go | 234 ++++++++++++++++++ internal/irc/irc.go | 191 +------------- internal/webserver/webserver.go | 166 ++++++++----- internal/webserver/webserver_test.go | 6 +- main.go | 31 ++- web/script.bat | 5 + 13 files changed, 515 insertions(+), 252 deletions(-) create mode 100755 deployments/helm/twitchsingstools/templates/deployment-db.yaml create mode 100755 deployments/helm/twitchsingstools/templates/pvc-db.yaml create mode 100755 deployments/helm/twitchsingstools/templates/service-db.yaml create mode 100755 internal/data/data.go create mode 100755 web/script.bat diff --git a/build/react-frontend/src/Components/AppAdmin/AppAdmin.js b/build/react-frontend/src/Components/AppAdmin/AppAdmin.js index 7f54de7..3c973e2 100644 --- a/build/react-frontend/src/Components/AppAdmin/AppAdmin.js +++ b/build/react-frontend/src/Components/AppAdmin/AppAdmin.js @@ -100,6 +100,7 @@ class AppAdmin extends React.Component { const adminToken = window.location.href.split("/")[5] const csvURL = "/csv/"+ channelName + "/" + adminToken const tsvURL = "/tsv/"+ channelName + "/" + adminToken + const scriptURL = "/script.bat/"+ channelName + "/" + adminToken const setPage = this.setPage const reloadDuetComponent = this.state.reloadDuetComponent const reloadSongComponent = this.state.reloadSongComponent @@ -121,9 +122,10 @@ class AppAdmin extends React.Component { - - - + + + + @@ -139,6 +141,26 @@ class AppAdmin extends React.Component { onReloadedChange={this.reloadDuetComponent}/> + + RIP Twitch sings! + + + Here is a script to download all your published duets and solo performances. + + + First, you need to download youtube-dl from here. + Then, download your personal script from here.

+ + + Put both files in the same folder and run script.bat. I know, that's not a nice interface, but, it works. Right now, I can't bear to do much more, + hurts to even think about the situation. It creates a subfolder per person and dates the files. + It even works if you sang the same song with the same person on the same day.

+ + + If you need to pause, press "Control" and "C" and say "Y" to the question. Next run the script will actually happily resume where it left off. + + + Download your data as : diff --git a/deployments/helm/twitchsingstools/templates/deployment-db.yaml b/deployments/helm/twitchsingstools/templates/deployment-db.yaml new file mode 100755 index 0000000..2d82ff8 --- /dev/null +++ b/deployments/helm/twitchsingstools/templates/deployment-db.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "twitchsingstools.fullname" . }}-db + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + labels: + {{- include "twitchsingstools.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "twitchsingstools.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "twitchsingstools.selectorLabels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "twitchsingstools.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.db.image.repository }}:{{ .Values.db.image.tag | default "latest" }}" + imagePullPolicy: {{ .Values.db.image.pullPolicy }} + ports: + - name: redis + containerPort: 9221 + protocol: TCP + livenessProbe: + tcpSocket: + port: redis + readinessProbe: + tcpSocket: + port: redis + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - mountPath: /pika/db + name: data + volumes: + - name: data + persistentVolumeClaim: + claimName: tstools-db + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployments/helm/twitchsingstools/templates/deployment.yaml b/deployments/helm/twitchsingstools/templates/deployment.yaml index d31112f..95394c2 100644 --- a/deployments/helm/twitchsingstools/templates/deployment.yaml +++ b/deployments/helm/twitchsingstools/templates/deployment.yaml @@ -32,6 +32,8 @@ spec: env: - name: TSTOOLS_DATA_FOLDER value: /data + - name: TSTOOLS_REDIS_HOST + value: {{ .Values.db.service.name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default "latest" }}" diff --git a/deployments/helm/twitchsingstools/templates/pvc-db.yaml b/deployments/helm/twitchsingstools/templates/pvc-db.yaml new file mode 100755 index 0000000..47e8acb --- /dev/null +++ b/deployments/helm/twitchsingstools/templates/pvc-db.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tstools-db + labels: + {{- include "twitchsingstools.labels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.db.storageSize }} diff --git a/deployments/helm/twitchsingstools/templates/service-db.yaml b/deployments/helm/twitchsingstools/templates/service-db.yaml new file mode 100755 index 0000000..03c7120 --- /dev/null +++ b/deployments/helm/twitchsingstools/templates/service-db.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.db.service.name }} + labels: + {{- include "twitchsingstools.labels" . | nindent 4 }} +spec: + type: {{ .Values.db.service.type }} + ports: + - port: {{ .Values.db.service.port }} + targetPort: redis + protocol: TCP + name: redis + selector: + {{- include "twitchsingstools.selectorLabels" . | nindent 4 }} diff --git a/deployments/helm/twitchsingstools/values-dev.yaml b/deployments/helm/twitchsingstools/values-dev.yaml index 88b14d8..af9b94b 100644 --- a/deployments/helm/twitchsingstools/values-dev.yaml +++ b/deployments/helm/twitchsingstools/values-dev.yaml @@ -6,7 +6,7 @@ replicaCount: 1 image: repository: imartyn/twitchsingstools - pullPolicy: IfNotPresent + pullPolicy: Always tag: dev imagePullSecrets: [] diff --git a/deployments/helm/twitchsingstools/values.yaml b/deployments/helm/twitchsingstools/values.yaml index 40cab24..fff8f69 100644 --- a/deployments/helm/twitchsingstools/values.yaml +++ b/deployments/helm/twitchsingstools/values.yaml @@ -41,6 +41,15 @@ twitchapp: {} storageSize: 10Gi +db: + service: + port: 4920 + name: tstools-db + storageSize: 10Gi + image: + repository: pikadb/pika + pullPolicy: IfNotPresent + service: type: ClusterIP port: 80 diff --git a/internal/data/data.go b/internal/data/data.go new file mode 100755 index 0000000..12050a1 --- /dev/null +++ b/internal/data/data.go @@ -0,0 +1,234 @@ +package data + +import ( + "encoding/base64" + "encoding/json" + "errors" + "log" + "os" + "time" + + rgb "github.com/foresthoffman/rgblog" + redis "github.com/gomodule/redigo/redis" + uuid "github.com/google/uuid" +) + +// ConfigStruct is the base for the config file +type ConfigStruct struct { + InitialChannels []string `json:"channels"` + IrcOAuthPath string `json:"ircoauthpath,omitempty"` + StringPath string `json:"authpath,omitempty"` + DataPath string `json:"datapath,omitempty"` + ExternalURL string `json:"externalurl,omitempty"` + AppOAuthPath string `json:"appoauthpath,omitempty"` + DatabaseSVC string `json:"databasesvc,omitempty"` +} + +// CommandType Kinda an enum +type CommandType string + +// CommandType literals +const ( + RandomSinger CommandType = "RandomSinger" + RandomPrompt CommandType = "RandomPrompt" + RandomSong CommandType = "RandomSong" + AgingSinger CommandType = "AgingSinger" + AgingSong CommandType = "AgingSong" +) + +// IsValid Is CommandType a valid enum? +func (ct CommandType) IsValid() error { + switch ct { + case RandomSinger, RandomPrompt, RandomSong, AgingSinger, AgingSong: + return nil + } + return errors.New("Invalid command type") +} + +// ChannelData is what we store in the BitRaft (Redis) database +type ChannelData struct { + ControlChannel bool + Name string `json:"name"` + AdminKey string `json:"value,omitempty"` + Commands []CommandStruct `json:"commands,omitempty"` + ExtraStrings string `json:"extrastrings,omitempty"` + JoinTime time.Time `json:"jointime"` + HasLeft bool `json:"hasleft"` + VideoCache []SingsVideoStruct `json:"videoCache"` + VideoCacheUpdated time.Time `json:"videoCacheUpdated"` + Bearer string `json:"bearer"` + TwitchUserID string `json:"twitchUserID"` +} + +// SingsVideoStruct The data we pull from Twitch +type SingsVideoStruct struct { + Date time.Time `json:"date"` // Golang date of creation + FullTitle string `json:"fullTitle"` // Full Title + Duet bool `json:"duet"` // Is it a duet? + OtherSinger string `json:"otherSinger"` // Twitch NAME of the other singer, extracted from the title + SongTitle string `json:"songTitle"` // extracted from title + LastSungSong time.Time `json:"LastSungSong"` // Last time this SONG was sung + LastSungSinger time.Time `json:"LastSungSinger"` // Last time a duet was sung with this SINGER, regardless of song, only Duets have this date initialised + VideoURL string `json:"VideoURL"` // RIP Twitch Sings +} + +// CommandStruct keypair for irc command -> actual thing to do +type CommandStruct struct { + CommandName CommandType `json:"commandName"` + KeyWord string `json:"keyword"` +} + +// GlobalData Some kind of architect would kill me for this +type GlobalData struct { + ChannelData map[string]ChannelData + Config ConfigStruct + Database redis.Conn + ControlChannel string +} + +// ConnectDatabase Connects to the database set in the config struct +func (gd *GlobalData) ConnectDatabase() { + var err error + rgb.YPrintf("[%s] Connecting to \"redis\" %s...\n", TimeStamp(), gd.Config.DatabaseSVC, err) + gd.Database, err = redis.Dial("tcp", gd.Config.DatabaseSVC+":4920") + if err != nil { + rgb.RPrintf("[%s] Failed connecting to \"redis\" %s : %s\n", TimeStamp(), gd.Config.DatabaseSVC, err) + os.Exit(1) + } + rgb.YPrintf("[%s] No error... wtf?\n", TimeStamp()) +} + +// UpdateVideoCache Updates the in-memory data and updates redis +func (gd *GlobalData) UpdateVideoCache(user string, videos []SingsVideoStruct) { + record := gd.ChannelData[user] + rgb.YPrintf("Replacing cache of %d performances with a new cache of %d performances\n", len(record.VideoCache), len(videos)) + record.VideoCache = videos + record.VideoCacheUpdated = time.Now() + asJson, _ := json.Marshal(record) + _, err := gd.Database.Do("SET", user, asJson) + if err != nil { + log.Fatal(err) + } + gd.ChannelData[user] = record +} + +func (gd *GlobalData) UpdateBearerToken(user string, token string) { + record := gd.ChannelData[user] + record.Bearer = token + asJson, _ := json.Marshal(record) + gd.Database.Do("SET", user, asJson) + gd.ChannelData[user] = record +} + +func (gd *GlobalData) UpdateTwitchUserID(user string, userid string) { + record := gd.ChannelData[user] + record.TwitchUserID = userid + asJson, _ := json.Marshal(record) + gd.Database.Do("SET", user, asJson) + gd.ChannelData[user] = record +} + +func (gd *GlobalData) UpdateChannelKey(user string, channelKey string) { + record := gd.ChannelData[user] + record.AdminKey = channelKey + asJson, _ := json.Marshal(record) + gd.Database.Do("SET", user, asJson) + gd.ChannelData[user] = record +} + +func (gd *GlobalData) UpdateChannelName(user string, newName string) { + record := gd.ChannelData[user] + record.Name = newName + asJson, _ := json.Marshal(record) + gd.Database.Do("SET", newName, asJson) + gd.ChannelData[newName] = record + //dunno why we'd need this but I guess in case? + if newName != user { + delete(gd.ChannelData, user) + gd.Database.Do("DEL", newName) + } +} + +func (gd *GlobalData) UpdateJoined(user string, invert bool) { + record := gd.ChannelData[user] + + if record.Name == "" { + record = ChannelData{Name: user, JoinTime: time.Now(), Commands: nil, ControlChannel: true} + } + record.JoinTime = time.Now() + asJson, _ := json.Marshal(record) + if invert { + record.HasLeft = true + } else { + record.HasLeft = false + } + gd.Database.Do("SET", user, asJson) + gd.ChannelData[user] = record +} + +func (gd *GlobalData) ReadChannelData() error { + keys, err := redis.Strings(gd.Database.Do("KEYS", "*")) + if err != nil { + rgb.RPrintf("[%s] ERROR with redis fetch : %s\n", TimeStamp(), err.Error()) + rgb.YPrintf("[%s] Maybe an empty redis, creating a record...\n", TimeStamp()) + keys = []string{} + } + if len(keys) == 0 { + rgb.YPrintf("[%s] Looks like an empty redis, creating a record...\n", TimeStamp()) + record := ChannelData{Name: gd.ControlChannel, JoinTime: time.Now(), Commands: nil} + asJSON, _ := json.Marshal(record) + gd.Database.Do("SET", gd.ControlChannel, asJSON) + keys = []string{gd.ControlChannel} + } + rgb.YPrintf("[%s] \"redis\" has %d records!\n", TimeStamp(), len(keys)) + for _, channel := range keys { + fetchedData, err := redis.String(gd.Database.Do("GET", channel)) + if err != nil { + rgb.YPrintf("[%s] failed to read key %s from redis, good luck!...\n", TimeStamp(), channel) + } + rgb.YPrintf("[%s] data from \"redis\" for %s is %s\n", TimeStamp(), channel, fetchedData) + cd := gd.ChannelData + if cd == nil { + cd = make(map[string]ChannelData) + } + d := &ChannelData{} + err = json.Unmarshal([]byte(fetchedData), d) + if err != nil { + rgb.RPrintf("[%s] channel data could not be unmarshalled : %s\n", TimeStamp(), err.Error()) + } + cd[channel] = *d + gd.ChannelData = cd + rgb.YPrintf("[%s] channel data : %v\n", TimeStamp(), gd.ChannelData) + } + // Managed to leave the main channel!? + rgb.YPrintf("[%s] Read channel data for %d channels\n", TimeStamp(), len(gd.ChannelData)) + return nil +} + +func (gd *GlobalData) ReadOrCreateChannelKey(channel string) string { + record := gd.ChannelData[channel] + magicCode := "" + if record.AdminKey == "" { + rgb.YPrintf("[%s] No channel key for #%s exists, creating one\n", TimeStamp(), channel) + newuu, _ := uuid.NewRandom() + magicCode = base64.StdEncoding.EncodeToString([]byte(newuu.String())) + gd.UpdateJoined(channel, true) + gd.UpdateChannelKey(channel, magicCode) + gd.UpdateChannelName(channel, channel) + rgb.YPrintf("[%s] Cached channel key for #%s\n", TimeStamp(), record.Name) + } else { + magicCode = record.AdminKey + rgb.YPrintf("[%s] Loaded data for #%s\n", TimeStamp(), channel) + } + return magicCode +} + +const UTCFormat = "Jan 2 15:04:05 UTC" + +func TimeStamp() string { + return TimeStampFmt(UTCFormat) +} + +func TimeStampFmt(format string) string { + return time.Now().Format(format) +} diff --git a/internal/irc/irc.go b/internal/irc/irc.go index 757507c..2e3129d 100644 --- a/internal/irc/irc.go +++ b/internal/irc/irc.go @@ -2,7 +2,6 @@ package irc import ( "bufio" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -15,9 +14,8 @@ import ( "strings" "time" + data "git.martyn.berlin/martyn/twitchsingstools/internal/data" rgb "github.com/foresthoffman/rgblog" - uuid "github.com/google/uuid" - scribble "github.com/nanobox-io/golang-scribble" ) const UTCFormat = "Jan 2 15:04:05 UTC" @@ -58,15 +56,6 @@ type AppOAuthCred struct { ClientSecret string `json:"client_secret,omitempty"` } -type ConfigStruct struct { - InitialChannels []string `json:"channels"` - IrcOAuthPath string `json:"ircoauthpath,omitempty"` - StringPath string `json:"authpath,omitempty"` - DataPath string `json:"datapath,omitempty"` - ExternalUrl string `json:"externalurl,omitempty"` - AppOAuthPath string `json:"appoauthpath,omitempty"` -} - type KardBot struct { Channel string conn net.Conn @@ -80,57 +69,7 @@ type KardBot struct { Server string startTime time.Time Prompts []string - Database scribble.Driver - ChannelData map[string]ChannelData - Config ConfigStruct -} - -type SingsVideoStruct struct { - Date time.Time `json:"date"` // Golang date of creation - FullTitle string `json:"fullTitle"` // Full Title - Duet bool `json:"duet"` // Is it a duet? - OtherSinger string `json:"otherSinger"` // Twitch NAME of the other singer, extracted from the title - SongTitle string `json:"songTitle"` // extracted from title - LastSungSong time.Time `json:"LastSungSong"` // Last time this SONG was sung - LastSungSinger time.Time `json:"LastSungSinger"` // Last time a duet was sung with this SINGER, regardless of song, only Duets have this date initialised -} - -// CommandType Kinda an enum -type CommandType string - -const ( - RandomSinger CommandType = "RandomSinger" - RandomPrompt CommandType = "RandomPrompt" - RandomSong CommandType = "RandomSong" - AgingSinger CommandType = "AgingSinger" - AgingSong CommandType = "AgingSong" -) - -func (ct CommandType) IsValid() error { - switch ct { - case RandomSinger, RandomPrompt, RandomSong, AgingSinger, AgingSong: - return nil - } - return errors.New("Invalid command type") -} - -type CommandStruct struct { - CommandName CommandType `json:"commandName"` - KeyWord string `json:"keyword"` -} - -type ChannelData struct { - ControlChannel bool - Name string `json:"name"` - AdminKey string `json:"value,omitempty"` - Commands []CommandStruct `json:"commands,omitempty"` - ExtraStrings string `json:"extrastrings,omitempty"` - JoinTime time.Time `json:"jointime"` - HasLeft bool `json:"hasleft"` - VideoCache []SingsVideoStruct `json:"videoCache"` - VideoCacheUpdated time.Time `json:"videoCacheUpdated"` - Bearer string `json:"bearer"` - TwitchUserID string `json:"twitchUserID"` + GlobalData data.GlobalData } // Connects the bot to the Twitch IRC server. The bot will continue to try to connect until it @@ -160,7 +99,7 @@ func (bb *KardBot) Disconnect() { // Look at the channels I'm actually in func (bb *KardBot) ActiveChannels() int { count := 0 - for _, channel := range bb.ChannelData { + for _, channel := range bb.GlobalData.ChannelData { if !channel.HasLeft { count = count + 1 } @@ -168,29 +107,6 @@ func (bb *KardBot) ActiveChannels() int { return count } -func (bb *KardBot) UpdateVideoCache(user string, videos []SingsVideoStruct) { - record := bb.ChannelData[user] - fmt.Printf("Replacing cache of %d performances with a new cache of %d performances", len(record.VideoCache), len(videos)) - record.VideoCache = videos - record.VideoCacheUpdated = time.Now() - bb.Database.Write("channelData", user, record) - bb.ChannelData[user] = record -} - -func (bb *KardBot) UpdateBearerToken(user string, token string) { - record := bb.ChannelData[user] - record.Bearer = token - bb.Database.Write("channelData", user, record) - bb.ChannelData[user] = record -} - -func (bb *KardBot) UpdateTwitchUserID(user string, userid string) { - record := bb.ChannelData[user] - record.TwitchUserID = userid - bb.Database.Write("channelData", user, record) - bb.ChannelData[user] = record -} - // Listens for and logs messages from chat. Responds to commands from the channel owner. The bot // continues until it gets disconnected, told to shutdown, or forcefully shutdown. func (bb *KardBot) HandleChat() error { @@ -222,11 +138,7 @@ func (bb *KardBot) HandleChat() error { matches := ConnectRegex.FindStringSubmatch(line) if nil != matches { realUserName := matches[1] - if bb.ChannelData[realUserName].Name == "" { - record := ChannelData{Name: realUserName, JoinTime: time.Now(), Commands: nil, ControlChannel: true} - bb.Database.Write("channelData", realUserName, record) - bb.ChannelData[realUserName] = record - } + bb.GlobalData.UpdateJoined(realUserName, false) bb.JoinChannel(realUserName) } @@ -258,13 +170,13 @@ func (bb *KardBot) HandleChat() error { if nil != cmdMatches { cmd := cmdMatches[1] cardCommand := "" - commands := bb.ChannelData[channel].Commands + commands := bb.GlobalData.ChannelData[channel].Commands for _, command := range commands { - if command.CommandName == RandomPrompt { + if command.CommandName == data.RandomPrompt { cardCommand = command.KeyWord } } - rgb.YPrintf("[%s] Checking cmd %s against [%s]\n", TimeStamp(), cmd, bb.ChannelData[channel].Commands) + rgb.YPrintf("[%s] Checking cmd %s against [%s]\n", TimeStamp(), cmd, bb.GlobalData.ChannelData[channel].Commands) switch cmd { case cardCommand: if cardCommand != "" { @@ -272,13 +184,9 @@ func (bb *KardBot) HandleChat() error { bb.Say("Your prompt is : "+bb.Prompts[rand.Intn(len(bb.Prompts))], channel) } case "join": - if bb.ChannelData[channel].ControlChannel { + if bb.GlobalData.ChannelData[channel].ControlChannel { rgb.CPrintf("[%s] Join asked for by %s on %s' channel!\n", TimeStamp(), userName, channel) - if bb.ChannelData[userName].Name == "" { - record := ChannelData{Name: userName, JoinTime: time.Now(), Commands: nil, ControlChannel: true} - bb.Database.Write("channelData", userName, record) - bb.ChannelData[userName] = record - } + bb.GlobalData.UpdateJoined(userName, false) bb.JoinChannel(userName) } } @@ -295,7 +203,7 @@ func (bb *KardBot) HandleChat() error { bb.Disconnect() return nil case "kcardadmin": - magicCode := bb.ReadOrCreateChannelKey(channel) + magicCode := bb.GlobalData.ReadOrCreateChannelKey(channel) rgb.CPrintf( "[%s] Magic code is %s - https://karaokards.ing.martyn.berlin/admin/%s/%s\n", TimeStamp(), @@ -455,18 +363,11 @@ func (bb *KardBot) Start() { return } - err = bb.readChannelData() - if nil != err { - fmt.Println(err) - fmt.Println("Aborting!") - return - } - for { bb.Connect() bb.Login() - if len(bb.ChannelData) > 0 { - for channelName, channelData := range bb.ChannelData { + if len(bb.GlobalData.ChannelData) > 0 { + for channelName, channelData := range bb.GlobalData.ChannelData { if !channelData.HasLeft { bb.JoinChannel(channelName) } @@ -487,74 +388,6 @@ func (bb *KardBot) Start() { } } -func (bb *KardBot) readChannelData() error { - records, err := bb.Database.ReadAll("channelData") - if err != nil { - // no db? initialise one? - record := ChannelData{Name: bb.Channel, JoinTime: time.Now(), Commands: nil} - rgb.YPrintf("[%s] No channel table for #%s exists, creating...\n", TimeStamp(), bb.Channel) - if err := bb.Database.Write("channelData", bb.Channel, record); err != nil { - return err - } - bb.ChannelData = make(map[string]ChannelData) - bb.ChannelData[bb.Channel] = record - } else { - bb.ChannelData = make(map[string]ChannelData) - } - for _, data := range records { - record := ChannelData{} - err := json.Unmarshal([]byte(data), &record) - if err != nil { - return err - } - } - // Managed to leave the main channel!? - if bb.ChannelData[bb.Channel].Name == "" { - rgb.YPrintf("[%s] No channel data for #%s exists, creating...\n", TimeStamp(), bb.Channel) - record := ChannelData{Name: bb.Channel, JoinTime: time.Now(), Commands: nil} - bb.ChannelData[bb.Channel] = record - if err := bb.Database.Write("channelData", bb.Channel, record); err != nil { - return err - } - records, err = bb.Database.ReadAll("channelData") - } - rgb.YPrintf("[%s] Read channel data for %d channels\n", TimeStamp(), len(bb.ChannelData)) - return nil -} - -func (bb *KardBot) ReadOrCreateChannelKey(channel string) string { - magicCode := "" - var err error - var record ChannelData - if record, ok := bb.ChannelData[channel]; !ok { - rgb.YPrintf("[%s] No channel data for #%s exists, creating\n", TimeStamp(), channel) - err = bb.Database.Read("channelData", channel, &record) - if err == nil { - bb.ChannelData[channel] = record - } - } - record = bb.ChannelData[channel] - if err != nil || record.AdminKey == "" { - rgb.YPrintf("[%s] No channel key for #%s exists, creating one\n", TimeStamp(), channel) - newuu, _ := uuid.NewRandom() - magicCode = base64.StdEncoding.EncodeToString([]byte(newuu.String())) - record.HasLeft = true - record.AdminKey = magicCode - if record.Name == "" { - record.Name = channel - } - if err := bb.Database.Write("channelData", channel, record); err != nil { - rgb.RPrintf("[%s] Error writing channel data for #%s\n", TimeStamp(), channel) - } - bb.ChannelData[record.Name] = record - rgb.YPrintf("[%s] Cached channel key for #%s\n", TimeStamp(), record.Name) - } else { - magicCode = record.AdminKey - rgb.YPrintf("[%s] Loaded data for #%s\n", TimeStamp(), channel) - } - return magicCode -} - func TimeStamp() string { return TimeStampFmt(UTCFormat) } diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go index 1817ef5..6ea918d 100755 --- a/internal/webserver/webserver.go +++ b/internal/webserver/webserver.go @@ -9,6 +9,7 @@ import ( "sync" "unicode" + data "git.martyn.berlin/martyn/twitchsingstools/internal/data" irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc" "github.com/dustin/go-humanize" "github.com/gorilla/handlers" @@ -77,6 +78,7 @@ type videosResponse struct { } var ircBot *irc.KardBot +var globalData *data.GlobalData func HealthHandler(response http.ResponseWriter, request *http.Request) { response.Header().Add("Content-type", "text/plain") @@ -130,7 +132,7 @@ func TemplateHandler(response http.ResponseWriter, request *http.Request) { // NotFoundHandler(response, request) // return } - var td = TemplateData{ircBot.Prompts[rand.Intn(len(ircBot.Prompts))], len(ircBot.Prompts), ircBot.ActiveChannels(), 0, ircBot.AppCredentials.ClientID, "https://" + ircBot.Config.ExternalUrl} + var td = TemplateData{ircBot.Prompts[rand.Intn(len(ircBot.Prompts))], len(ircBot.Prompts), ircBot.ActiveChannels(), 0, ircBot.AppCredentials.ClientID, "https://" + globalData.Config.ExternalURL} err = tmpl.Execute(response, td) if err != nil { http.Error(response, err.Error(), http.StatusInternalServerError) @@ -153,6 +155,7 @@ func humanTimeFromTimeString(s string) string { type AugmentedSingsVideoStruct struct { Date time.Time NiceDate string + ShortDate string FullTitle string Duet bool OtherSinger string @@ -161,12 +164,15 @@ type AugmentedSingsVideoStruct struct { NiceLastSungSong string LastSungSinger time.Time NiceLastSungSinger string + VideoURL string + VideoNumber string //yes, I don't care any more. } -func AugmentSingsVideoStructForCSV(input irc.SingsVideoStruct) AugmentedSingsVideoStruct { +func AugmentSingsVideoStructForCSV(input data.SingsVideoStruct) AugmentedSingsVideoStruct { var ret AugmentedSingsVideoStruct ret.Date = input.Date ret.NiceDate = input.Date.Format("2006-01-02 15:04:05") + ret.ShortDate = input.Date.Format("2006-01-02") ret.FullTitle = input.FullTitle ret.Duet = input.Duet ret.OtherSinger = input.OtherSinger @@ -174,6 +180,9 @@ func AugmentSingsVideoStructForCSV(input irc.SingsVideoStruct) AugmentedSingsVid ret.LastSungSong = input.LastSungSong ret.NiceLastSungSong = input.LastSungSong.Format("2006-01-02 15:04:05") ret.LastSungSinger = input.LastSungSinger + ret.VideoURL = input.VideoURL + urlParts := strings.Split(input.VideoURL, "/") + ret.VideoNumber = urlParts[len(urlParts)-1] if !ret.Duet { ret.NiceLastSungSinger = "Solo performance" } else { @@ -182,7 +191,7 @@ func AugmentSingsVideoStructForCSV(input irc.SingsVideoStruct) AugmentedSingsVid return ret } -func AugmentSingsVideoStructSliceForCSV(input []irc.SingsVideoStruct) []AugmentedSingsVideoStruct { +func AugmentSingsVideoStructSliceForCSV(input []data.SingsVideoStruct) []AugmentedSingsVideoStruct { ret := make([]AugmentedSingsVideoStruct, 0) for _, record := range input { ret = append(ret, AugmentSingsVideoStructForCSV(record)) @@ -190,7 +199,7 @@ func AugmentSingsVideoStructSliceForCSV(input []irc.SingsVideoStruct) []Augmente return ret } -func AugmentSingsVideoStruct(input irc.SingsVideoStruct) AugmentedSingsVideoStruct { +func AugmentSingsVideoStruct(input data.SingsVideoStruct) AugmentedSingsVideoStruct { var ret AugmentedSingsVideoStruct ret.Date = input.Date ret.NiceDate = humanize.Time(input.Date) @@ -209,7 +218,7 @@ func AugmentSingsVideoStruct(input irc.SingsVideoStruct) AugmentedSingsVideoStru return ret } -func AugmentSingsVideoStructSlice(input []irc.SingsVideoStruct) []AugmentedSingsVideoStruct { +func AugmentSingsVideoStructSlice(input []data.SingsVideoStruct) []AugmentedSingsVideoStruct { ret := make([]AugmentedSingsVideoStruct, 0) for _, record := range input { ret = append(ret, AugmentSingsVideoStruct(record)) @@ -266,8 +275,8 @@ func ValidateTwitchBearerToken(bearer string) (bool, error) { return resp.StatusCode == 200, nil } -func twitchVidToSingsVid(twitchFormat videoStruct) (irc.SingsVideoStruct, error) { - var ret irc.SingsVideoStruct +func twitchVidToSingsVid(twitchFormat videoStruct) (data.SingsVideoStruct, error) { + var ret data.SingsVideoStruct layout := "2006-01-02T15:04:05Z" var d time.Time d, err := time.Parse(layout, twitchFormat.CreatedAt) @@ -275,6 +284,7 @@ func twitchVidToSingsVid(twitchFormat videoStruct) (irc.SingsVideoStruct, error) return ret, err } ret.Date = d + ret.VideoURL = twitchFormat.URL var DuetRegex = regexp.MustCompile(`^Duet with ([^ ]*): (.*)$`) matches := DuetRegex.FindAllStringSubmatch(twitchFormat.Title, -1) @@ -307,7 +317,7 @@ type SingerSings struct { Sings int } -func calculateTopNSongs(songCache []irc.SingsVideoStruct, howMany int) []SongSings { +func calculateTopNSongs(songCache []data.SingsVideoStruct, howMany int) []SongSings { songMap := map[string]int{} for _, record := range songCache { sings := songMap[record.SongTitle] @@ -338,7 +348,7 @@ func IsLower(s string) bool { return true } -func unmangleSingerName(MangledCaseName string, songCache []irc.SingsVideoStruct) string { +func unmangleSingerName(MangledCaseName string, songCache []data.SingsVideoStruct) string { options := make(map[string]string, 0) for _, record := range songCache { if strings.ToUpper(MangledCaseName) == strings.ToUpper(record.OtherSinger) { @@ -413,7 +423,7 @@ type CacheDetails struct { func getCacheDetails(channel string) CacheDetails { var ret CacheDetails - channelData := ircBot.ChannelData[channel] + channelData := globalData.ChannelData[channel] ret.Age = time.Now().Sub(channelData.VideoCacheUpdated) ret.AgeStr = humanize.Time(channelData.VideoCacheUpdated) ret.SongCount = len(channelData.VideoCache) @@ -422,36 +432,36 @@ func getCacheDetails(channel string) CacheDetails { func forceUpdateCache(channel string) { fmt.Printf("Forcing cache update!") - channelData := ircBot.ChannelData[channel] + channelData := globalData.ChannelData[channel] tenHours := time.Hour * -10 videoCacheUpdated := time.Now().Add(tenHours) // Subtract 10 hours from now, cache is 10 hours old. channelData.VideoCacheUpdated = videoCacheUpdated - ircBot.ChannelData[channel] = channelData + globalData.ChannelData[channel] = channelData updateCacheIfNecessary(channel) } func updateCacheIfNecessary(channel string) { cacheLock.Lock() - channelData := ircBot.ChannelData[channel] + channelData := globalData.ChannelData[channel] if time.Now().Sub(channelData.VideoCacheUpdated).Hours() > 1 { - fmt.Printf("Cache of %d performances is older than an hour - %.1f hours old to be precise... fetching.\n", len(channelData.VideoCache), time.Now().Sub(ircBot.ChannelData[channel].VideoCacheUpdated).Hours()) + fmt.Printf("Cache of %d performances is older than an hour - %.1f hours old to be precise... fetching.\n", len(channelData.VideoCache), time.Now().Sub(globalData.ChannelData[channel].VideoCacheUpdated).Hours()) vids, err := fetchAllVoDs(channelData.TwitchUserID, channelData.Bearer) if err != nil { - errCache := make([]irc.SingsVideoStruct, 0) - var ret irc.SingsVideoStruct + errCache := make([]data.SingsVideoStruct, 0) + var ret data.SingsVideoStruct ret.FullTitle = "Error fetching videos: " + err.Error() errCache = append(errCache, ret) vids = errCache } updateCalculatedFields(vids) - ircBot.UpdateVideoCache(channel, vids) + globalData.UpdateVideoCache(channel, vids) } else { - fmt.Printf("Cache of %d performances is younger than an hour - %.1f hours old to be precise... not fetching.\n", len(channelData.VideoCache), time.Now().Sub(ircBot.ChannelData[channel].VideoCacheUpdated).Hours()) + fmt.Printf("Cache of %d performances is younger than an hour - %.1f hours old to be precise... not fetching.\n", len(channelData.VideoCache), time.Now().Sub(globalData.ChannelData[channel].VideoCacheUpdated).Hours()) } cacheLock.Unlock() } -func calculateTopNSingers(songCache []irc.SingsVideoStruct, howMany int) []SingerSings { +func calculateTopNSingers(songCache []data.SingsVideoStruct, howMany int) []SingerSings { songMap := map[string]int{} songCount := 0 for _, record := range songCache { @@ -475,7 +485,7 @@ func calculateTopNSingers(songCache []irc.SingsVideoStruct, howMany int) []Singe return slice } -func calculateLastSungSongDate(songCache []irc.SingsVideoStruct, SongTitle string) time.Time { +func calculateLastSungSongDate(songCache []data.SingsVideoStruct, SongTitle string) time.Time { var t time.Time for _, record := range songCache { if record.SongTitle == SongTitle { @@ -487,7 +497,7 @@ func calculateLastSungSongDate(songCache []irc.SingsVideoStruct, SongTitle strin return t } -func calculateLastSungSingerDate(songCache []irc.SingsVideoStruct, Singer string) time.Time { +func calculateLastSungSingerDate(songCache []data.SingsVideoStruct, Singer string) time.Time { var t time.Time for _, record := range songCache { if strings.ToUpper(record.OtherSinger) == strings.ToUpper(Singer) { @@ -502,7 +512,7 @@ func calculateLastSungSingerDate(songCache []irc.SingsVideoStruct, Singer string return t } -func updateCalculatedFields(songCache []irc.SingsVideoStruct) { +func updateCalculatedFields(songCache []data.SingsVideoStruct) { for i, record := range songCache { if record.Duet { songCache[i].LastSungSinger = calculateLastSungSingerDate(songCache, record.OtherSinger) @@ -511,7 +521,7 @@ func updateCalculatedFields(songCache []irc.SingsVideoStruct) { } } -func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.SingsVideoStruct, error) { +func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]data.SingsVideoStruct, error) { url := "" if from == "" { url = "videos?user_id=" + userID + "&first=100&type=upload" @@ -529,7 +539,7 @@ func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.S if err != nil { return nil, err } - titles := make([]irc.SingsVideoStruct, 0) + titles := make([]data.SingsVideoStruct, 0) for _, videoData := range fullResponse.Data { ret, err := twitchVidToSingsVid(videoData) if err != nil { @@ -549,7 +559,7 @@ func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.S return titles, nil } -func fetchAllVoDs(userID string, bearer string) ([]irc.SingsVideoStruct, error) { +func fetchAllVoDs(userID string, bearer string) ([]data.SingsVideoStruct, error) { tokenValid, err := ValidateTwitchBearerToken(bearer) if err != nil { fmt.Println("Error validating token : " + err.Error()) @@ -561,7 +571,7 @@ func fetchAllVoDs(userID string, bearer string) ([]irc.SingsVideoStruct, error) } titles, err := fetchVoDsPagesRecursive(userID, bearer, "") if err != nil { - return make([]irc.SingsVideoStruct, 0), err + return make([]data.SingsVideoStruct, 0), err } return titles, nil } @@ -578,7 +588,7 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) { "client_secret": {ircBot.AppCredentials.ClientSecret}, "code": {vars["code"]}, "grant_type": {"authorization_code"}, - "redirect_uri": {"https://" + ircBot.Config.ExternalUrl + "/twitchadmin"}}) + "redirect_uri": {"https://" + globalData.Config.ExternalURL + "/twitchadmin"}}) if err != nil { response.WriteHeader(500) response.Header().Add("Content-type", "text/plain") @@ -629,10 +639,10 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) { } user := usersObject.Data[0] - magicCode := ircBot.ReadOrCreateChannelKey(user.Login) - ircBot.UpdateBearerToken(user.Login, oauthResponse.Access_token) - ircBot.UpdateTwitchUserID(user.Login, user.Id) - url := "https://" + ircBot.Config.ExternalUrl + "/admin/" + user.Login + "/" + magicCode + magicCode := globalData.ReadOrCreateChannelKey(user.Login) + globalData.UpdateBearerToken(user.Login, oauthResponse.Access_token) + globalData.UpdateTwitchUserID(user.Login, user.Id) + url := "https://" + globalData.Config.ExternalURL + "/admin/" + user.Login + "/" + magicCode http.Redirect(response, request, url, http.StatusFound) } else { @@ -664,7 +674,7 @@ func TwitchBackendHandler(response http.ResponseWriter, request *http.Request) { // ircBot.AppCredentials.Password // vars["oauthtoken"] // authorization_code - // "https://"+ircBot.Config.ExternalUrl+/twitchadmin + // "https://"+globalData.Config.ExternalURL+/twitchadmin fmt.Println("Asking twitch for more...") resp, err := http.PostForm( "https://id.twitch.tv/oauth2/token", @@ -673,7 +683,7 @@ func TwitchBackendHandler(response http.ResponseWriter, request *http.Request) { "client_secret": {ircBot.AppCredentials.ClientSecret}, "code": {vars["code"]}, "grant_type": {"authorization_code"}, - "redirect_uri": {"https://" + ircBot.Config.ExternalUrl + "/twitchadmin"}}) + "redirect_uri": {"https://" + globalData.Config.ExternalURL + "/twitchadmin"}}) if err != nil { response.Header().Add("Content-type", "text/plain") fmt.Fprint(response, "ERROR: "+err.Error()) @@ -693,13 +703,13 @@ func TwitchBackendHandler(response http.ResponseWriter, request *http.Request) { func CSVHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) - if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey { + if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey { UnauthorizedHandler(response, request) return } type TemplateData struct { Channel string - Commands []irc.CommandStruct + Commands []data.CommandStruct ExtraStrings string SinceTime time.Time SinceTimeUTC string @@ -710,7 +720,7 @@ func CSVHandler(response http.ResponseWriter, request *http.Request) { TopNSingers []SingerSings } updateCacheIfNecessary(vars["channel"]) - channelData := ircBot.ChannelData[vars["channel"]] + channelData := globalData.ChannelData[vars["channel"]] topNSongs := calculateTopNSongs(channelData.VideoCache, 10) topNSingers := calculateTopNSingers(channelData.VideoCache, 10) var td = TemplateData{channelData.Name, channelData.Commands, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft, AugmentSingsVideoStructSliceForCSV(channelData.VideoCache), topNSongs, topNSingers} @@ -729,13 +739,14 @@ func CSVHandler(response http.ResponseWriter, request *http.Request) { func JSONHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) - if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey { + if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey { + fmt.Printf("%s != %s\n", vars["key"], globalData.ChannelData[vars["channel"]].AdminKey) UnauthorizedHandler(response, request) return } type TemplateData struct { Channel string - Commands []irc.CommandStruct + Commands []data.CommandStruct ExtraStrings string SinceTime time.Time SinceTimeUTC string @@ -746,7 +757,7 @@ func JSONHandler(response http.ResponseWriter, request *http.Request) { TopNSingers []SingerSings } updateCacheIfNecessary(vars["channel"]) - channelData := ircBot.ChannelData[vars["channel"]] + channelData := globalData.ChannelData[vars["channel"]] var topNSongs []SongSings var topNSingers []SingerSings @@ -769,9 +780,45 @@ func JSONHandler(response http.ResponseWriter, request *http.Request) { } } +func ScriptHandler(response http.ResponseWriter, request *http.Request) { + vars := mux.Vars(request) + if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey { + UnauthorizedHandler(response, request) + return + } + type TemplateData struct { + Channel string + Commands []data.CommandStruct + ExtraStrings string + SinceTime time.Time + SinceTimeUTC string + Leaving bool + HasLeft bool + SongData []AugmentedSingsVideoStruct + TopNSongs []SongSings + TopNSingers []SingerSings + } + updateCacheIfNecessary(vars["channel"]) + channelData := globalData.ChannelData[vars["channel"]] + topNSongs := calculateTopNSongs(channelData.VideoCache, 10) + topNSingers := calculateTopNSingers(channelData.VideoCache, 10) + var td = TemplateData{channelData.Name, channelData.Commands, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft, AugmentSingsVideoStructSliceForCSV(channelData.VideoCache), topNSongs, topNSingers} + if request.URL.Path[0:11] == "/script.bat" { + response.Header().Add("Content-Disposition", "attachment; filename=\"script.bat\"") + response.Header().Add("Content-type", "application/x-bat") + tmpl := template.Must(template.ParseFiles("web/script.bat")) + tmpl.Execute(response, td) + } else { + response.Header().Add("Content-Disposition", "attachment; filename=\"script.sh\"") + response.Header().Add("Content-type", "text/x-shellscript") + tmpl := template.Must(template.ParseFiles("web/script.sh")) + tmpl.Execute(response, td) + } +} + func CacheDetailsHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) - if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey { + if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey { UnauthorizedHandler(response, request) return } @@ -783,23 +830,23 @@ func CacheDetailsHandler(response http.ResponseWriter, request *http.Request) { func BotDetailsHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) - if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey { + if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey { UnauthorizedHandler(response, request) return } type ChannelDataSmaller struct { - Commands []irc.CommandStruct `json:"commands,omitempty"` - ExtraStrings string `json:"extrastrings,omitempty"` - JoinTime time.Time `json:"jointime"` - HasLeft bool `json:"hasleft"` - VideoCacheUpdated time.Time `json:"videoCacheUpdated"` + Commands []data.CommandStruct `json:"commands,omitempty"` + ExtraStrings string `json:"extrastrings,omitempty"` + JoinTime time.Time `json:"jointime"` + HasLeft bool `json:"hasleft"` + VideoCacheUpdated time.Time `json:"videoCacheUpdated"` } var deets ChannelDataSmaller - deets.Commands = ircBot.ChannelData[vars["channel"]].Commands - deets.ExtraStrings = ircBot.ChannelData[vars["channel"]].ExtraStrings - deets.JoinTime = ircBot.ChannelData[vars["channel"]].JoinTime - deets.HasLeft = ircBot.ChannelData[vars["channel"]].HasLeft - deets.VideoCacheUpdated = ircBot.ChannelData[vars["channel"]].VideoCacheUpdated + deets.Commands = globalData.ChannelData[vars["channel"]].Commands + deets.ExtraStrings = globalData.ChannelData[vars["channel"]].ExtraStrings + deets.JoinTime = globalData.ChannelData[vars["channel"]].JoinTime + deets.HasLeft = globalData.ChannelData[vars["channel"]].HasLeft + deets.VideoCacheUpdated = globalData.ChannelData[vars["channel"]].VideoCacheUpdated response.Header().Add("Content-type", "application/json") enc := json.NewEncoder(response) enc.Encode(deets) @@ -815,7 +862,7 @@ func ReactIndexHandler(entrypoint string) func(w http.ResponseWriter, r *http.Re func ForceRefreshHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) - if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey { + if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey { UnauthorizedHandler(response, request) return } @@ -827,21 +874,17 @@ func ForceRefreshHandler(response http.ResponseWriter, request *http.Request) { func JoinHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) - if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey { + if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey { UnauthorizedHandler(response, request) return } - record := ircBot.ChannelData[vars["channel"]] - record.Name = vars["channel"] - record.JoinTime = time.Now() - record.HasLeft = false - ircBot.Database.Write("channelData", vars["channel"], record) - ircBot.ChannelData[vars["channel"]] = record - ircBot.JoinChannel(record.Name) + globalData.UpdateJoined(vars["channel"], false) + ircBot.JoinChannel(vars["channel"]) } -func HandleHTTP(passedIrcBot *irc.KardBot) { +func HandleHTTP(passedIrcBot *irc.KardBot, passedGlobalData *data.GlobalData) { ircBot = passedIrcBot + globalData = passedGlobalData r := mux.NewRouter() loggedRouter := handlers.LoggingHandler(os.Stdout, r) r.NotFoundHandler = http.HandlerFunc(NotFoundHandler) @@ -857,6 +900,7 @@ func HandleHTTP(passedIrcBot *irc.KardBot) { r.HandleFunc("/debug/{channel}/{key}", JSONHandler) r.HandleFunc("/topsongs/{channel}/{key}", JSONHandler) r.HandleFunc("/topsingers/{channel}/{key}", JSONHandler) + r.HandleFunc("/script.bat/{channel}/{key}", ScriptHandler) r.Path("/twitchtobackend").Queries("access_token", "{access_token}", "scope", "{scope}", "token_type", "{token_type}").HandlerFunc(TwitchBackendHandler) r.Path("/twitchadmin").Queries("code", "{code}", "scope", "{scope}").HandlerFunc(TwitchAdminHandler) r.PathPrefix("/static").Handler(http.StripPrefix("/static", http.FileServer(http.Dir("./web/react-frontend/static")))) diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index ba8c52c..6770ed7 100755 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc" + data "git.martyn.berlin/martyn/twitchsingstools/internal/data" ) func TestDateRegex(t *testing.T) { @@ -72,9 +72,9 @@ func TestSoloRegexes(t *testing.T) { } func TestCalculatedDates(t *testing.T) { - var record irc.SingsVideoStruct + var record data.SingsVideoStruct format := "2006-01-02 15:04:05 +0000 UTC" - var mockCache []irc.SingsVideoStruct + var mockCache []data.SingsVideoStruct record.Date, _ = time.Parse(format, "2020-07-13 19:43:11 +0000 UTC") record.SongTitle = "Words Fail" record.OtherSinger = "FullOfEmily" diff --git a/main.go b/main.go index 841427e..1278f4c 100755 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "io/ioutil" "math/rand" "os" @@ -9,6 +10,7 @@ import ( "time" builtins "git.martyn.berlin/martyn/twitchsingstools/internal/builtins" + data "git.martyn.berlin/martyn/twitchsingstools/internal/data" irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc" webserver "git.martyn.berlin/martyn/twitchsingstools/internal/webserver" rgb "github.com/foresthoffman/rgblog" @@ -23,7 +25,7 @@ var selectablePrompts []string var customStrings customStringsStruct -var config irc.ConfigStruct +var config data.ConfigStruct func readConfig() { var data []byte @@ -174,12 +176,32 @@ func main() { } } else { if _, err := os.Stat(config.AppOAuthPath); os.IsNotExist(err) { - rgb.YPrintf("[%s] Error config-specified oauth file %s doesn't exist, bailing!\n", irc.TimeStamp(), config.AppOAuthPath) + rgb.RPrintf("[%s] Error config-specified oauth file %s doesn't exist, bailing!\n", irc.TimeStamp(), config.AppOAuthPath) os.Exit(1) } appOauthPath = config.AppOAuthPath } + rgb.YPrintf("[%s] Starting connection to redis...\n", irc.TimeStamp()) + //TODO: unhardcode this + if os.Getenv("TSTOOLS_REDIS_HOST") != "" { + config.DatabaseSVC = os.Getenv("TSTOOLS_REDIS_HOST") + } else { + // assume localhost, which should fail. + config.DatabaseSVC = "localhost" + } + var globalData data.GlobalData + globalData.Config = config + globalData.ConnectDatabase() + defer globalData.Database.Close() + rgb.GPrintf("[%s] Connected to \"redis\" %s\n", irc.TimeStamp(), "config.DatabaseSVC") + err := globalData.ReadChannelData() + if nil != err { + fmt.Println(err) + fmt.Println("Aborting!") + os.Exit(1) + } + rgb.GPrintf("[%s] Read the channel data from \"redis\" successfully, now have %d records\n", irc.TimeStamp(), len(globalData.ChannelData)) // Replace the channel name, bot name, and the path to the private directory with your respective // values. var myBot irc.KardBot @@ -193,13 +215,12 @@ func main() { AppPrivatePath: appOauthPath, Server: "irc.chat.twitch.tv", Prompts: selectablePrompts, - Database: *persistentData, - Config: config, + GlobalData: globalData, } } go func() { rgb.YPrintf("[%s] Starting webserver on port %s\n", irc.TimeStamp(), "5353") - webserver.HandleHTTP(&myBot) + webserver.HandleHTTP(&myBot, &globalData) }() if ircOauthPath != "" { myBot.Start() diff --git a/web/script.bat b/web/script.bat new file mode 100755 index 0000000..a0cb24f --- /dev/null +++ b/web/script.bat @@ -0,0 +1,5 @@ +@ECHO OFF +{{ range .SongData -}} +IF NOT EXIST "{{.OtherSinger}}\" MKDIR {{.OtherSinger}} +IF NOT EXIST "{{.OtherSinger}}\{{.ShortDate}}-{{.OtherSinger}}-{{.SongTitle}}-{{.VideoNumber}}.mp4" youtube-dl -o "{{.OtherSinger}}\{{.ShortDate}}-{{.OtherSinger}}-{{.SongTitle}}-{{.VideoNumber}}.mp4" {{.VideoURL}} +{{ end }}