diff --git a/Makefile b/Makefile index 89f2aa2..c1d7692 100755 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ LDFLAGS=-ldflags "-X main.buildDate=${BUILD}" .PHONY: build deps static +test: + go test git.martyn.berlin/martyn/twitchsingstools/internal/webserver + build: go build ${LDFLAGS} diff --git a/build/ci/drone.yml b/build/ci/drone.yml index 5585f9b..1e7066e 100644 --- a/build/ci/drone.yml +++ b/build/ci/drone.yml @@ -14,8 +14,8 @@ steps: - mkdir -p /go/src/git.martyn.berlin/martyn - ln -s /drone/src /go/src/git.martyn.berlin/martyn/twitchsingstools - cd /go/src/git.martyn.berlin/martyn/twitchsingstools - - go get - - go build + - make test + - make - name: publish image: plugins/docker:18 @@ -54,8 +54,8 @@ steps: - mkdir -p /go/src/git.martyn.berlin/martyn - ln -s /drone/src /go/src/git.martyn.berlin/martyn/twitchsingstools - cd /go/src/git.martyn.berlin/martyn/twitchsingstools - - go get - - go build + - make test + - make trigger: ref: diff --git a/deployments/helm/twitchsingstools/templates/deployment.yaml b/deployments/helm/twitchsingstools/templates/deployment.yaml index edda388..d31112f 100644 --- a/deployments/helm/twitchsingstools/templates/deployment.yaml +++ b/deployments/helm/twitchsingstools/templates/deployment.yaml @@ -29,6 +29,9 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} + env: + - name: TSTOOLS_DATA_FOLDER + value: /data securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default "latest" }}" diff --git a/deployments/helm/twitchsingstools/values.yaml b/deployments/helm/twitchsingstools/values.yaml index 1e6f283..e382454 100644 --- a/deployments/helm/twitchsingstools/values.yaml +++ b/deployments/helm/twitchsingstools/values.yaml @@ -51,6 +51,7 @@ ingress: enabled: true annotations: kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt hosts: - host: twitchsingstools.ing.martyn.berlin paths: diff --git a/internal/irc/irc.go b/internal/irc/irc.go index 77423db..06487ed 100644 --- a/internal/irc/irc.go +++ b/internal/irc/irc.go @@ -85,14 +85,27 @@ type KardBot struct { Config ConfigStruct } +type SingsVideoStruct struct { + Date time.Time `json:"date"` // Golang date of creation + FullTitle string `json:"fullTitle"` // Full Title + Duet bool `json:"duet"` // Is it a duet? + OtherSinger string `json:"otherSinger"` // Twitch NAME of the other singer, extracted from the title + SongTitle string `json:"songTitle"` // extracted from title + LastSungSong time.Time `json:"LastSungSong"` // Last time this SONG was sung + LastSungSinger time.Time `json:"LastSungSinger"` // Last time a duet was sung with this SINGER, regardless of song, only Duets have this date initialised +} + type ChannelData struct { - Name string `json:"name"` - AdminKey string `json:"value,omitempty"` - Command string `json:"customcommand,omitempty"` - ExtraStrings string `json:"extrastrings,omitempty"` - JoinTime time.Time `json:"jointime"` - ControlChannel bool - HasLeft bool `json:"hasleft"` + Name string `json:"name"` + AdminKey string `json:"value,omitempty"` + Command string `json:"customcommand,omitempty"` + ExtraStrings string `json:"extrastrings,omitempty"` + JoinTime time.Time `json:"jointime"` + ControlChannel bool + HasLeft bool `json:"hasleft"` + VideoCache []SingsVideoStruct `json:"videoCache"` + VideoCacheUpdated time.Time `json:"videoCacheUpdated"` + Bearer string `json:"bearer"` } // Connects the bot to the Twitch IRC server. The bot will continue to try to connect until it @@ -130,6 +143,22 @@ func (bb *KardBot) ActiveChannels() int { return count } +func (bb *KardBot) UpdateVideoCache(user string, videos []SingsVideoStruct) { + record := bb.ChannelData[user] + fmt.Printf("Replacing cache of %d performances with a new cache of %d performances", len(record.VideoCache), len(videos)) + record.VideoCache = videos + record.VideoCacheUpdated = time.Now() + bb.Database.Write("channelData", user, record) + bb.ChannelData[user] = record +} + +func (bb *KardBot) UpdateBearerToken(user string, token string) { + record := bb.ChannelData[user] + record.Bearer = token + bb.Database.Write("channelData", user, record) + bb.ChannelData[user] = record +} + // Listens for and logs messages from chat. Responds to commands from the channel owner. The bot // continues until it gets disconnected, told to shutdown, or forcefully shutdown. func (bb *KardBot) HandleChat() error { diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go index 2a996db..bb214b9 100755 --- a/internal/webserver/webserver.go +++ b/internal/webserver/webserver.go @@ -1,9 +1,13 @@ package webserver import ( + "errors" "math/rand" + "regexp" + "sort" irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc" + "github.com/dustin/go-humanize" "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -44,6 +48,31 @@ type twitchUsersBigResponse struct { Data []twitchUser `json:"data"` } +type twitchCursor struct { + Cursor string `json:"cursor"` +} +type videoStruct struct { + CreatedAt string `json:"created_at"` // Date when the video was created. + Description string `json:"description"` // Description of the video. + Duration string `json:"duration"` // Length of the video. + ID string `json:"id"` // ID of the video. + Language string `json:"language"` // Language of the video. + Pagination string `json:"pagination"` // A cursor value, to be used in a subsequent request to specify the starting point of the next set of results. + PublishedAt string `json:"published_at"` // Date when the video was published. + ThumbnailURL string `json:"thumbnail_url"` // Template URL for the thumbnail of the video. + Title string `json:"title"` // Title of the video. + Type string `json:"type"` // Type of video. Valid values: "upload", "archive", "highlight". + URL string `json:"url"` // URL of the video. + UserID string `json:"user_id"` // ID of the user who owns the video. + UserName string `json:"user_name"` // Display name corresponding to user_id. + ViewCount int `json:"view_count"` // Number of times the video has been viewed. + Viewable string `json:"viewable"` // Indicates whether the video is publicly viewable. Valid values: "public", "private". +} +type videosResponse struct { + Data []videoStruct `json:"data"` + Pagination twitchCursor `json:"pagination"` +} + var ircBot *irc.KardBot func HealthHandler(response http.ResponseWriter, request *http.Request) { @@ -115,6 +144,13 @@ func LeaveHandler(response http.ResponseWriter, request *http.Request) { TemplateHandler(response, request) } +func humanTimeFromTimeString(s string) string { + return s + // format := "2006-01-02 15:04:05 +0000 UTC" + // d, _ := time.Parse(format, s) + // return humanize.Time(d) +} + func AdminHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey { @@ -129,9 +165,12 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) { SinceTimeUTC string Leaving bool HasLeft bool + SongData []irc.SingsVideoStruct + TopNSongs []SongSings } channelData := ircBot.ChannelData[vars["channel"]] - var td = TemplateData{channelData.Name, channelData.Command, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft} + topNSongs := calculateTopNSongs(channelData.VideoCache, 10) + var td = TemplateData{channelData.Name, channelData.Command, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft, channelData.VideoCache, topNSongs} if request.Method == "POST" { request.ParseForm() @@ -157,7 +196,7 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) { } ircBot.Database.Write("channelData", vars["channel"], record) ircBot.ChannelData[vars["channel"]] = record - td = TemplateData{record.Name, record.Command, record.ExtraStrings, record.JoinTime, record.JoinTime.Format(irc.UTCFormat), false, record.HasLeft} + td = TemplateData{record.Name, record.Command, record.ExtraStrings, record.JoinTime, record.JoinTime.Format(irc.UTCFormat), false, record.HasLeft, channelData.VideoCache, topNSongs} ircBot.JoinChannel(record.Name) } sourceData := ircBot.ChannelData[vars["channel"]] @@ -173,7 +212,12 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) { } ircBot.Database.Write("channelData", vars["channel"], sourceData) } - tmpl := template.Must(template.ParseFiles("web/admin.html")) + tmpl, err := template.New("admin.html").Funcs(template.FuncMap{ + "niceDate": humanize.Time, + }).ParseFiles("web/admin.html") + if err != nil { + panic(err.Error()) + } tmpl.Execute(response, td) } @@ -203,6 +247,174 @@ func twitchHTTPClient(call string, bearer string) (string, error) { return string([]byte(body)), nil } +func ValidateTwitchBearerToken(bearer string) (bool, error) { + url := "https://id.twitch.tv/oauth2/validate" + var bearerHeader = "Bearer " + bearer + + req, err := http.NewRequest("GET", url, nil) + req.Header.Add("Client-ID", ircBot.AppCredentials.ClientID) + req.Header.Add("Authorization", bearerHeader) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + + _, _ = ioutil.ReadAll(resp.Body) + return resp.StatusCode == 200, nil +} + +func twitchVidToSingsVid(twitchFormat videoStruct) (irc.SingsVideoStruct, error) { + var ret irc.SingsVideoStruct + layout := "2006-01-02T15:04:05Z" + var d time.Time + d, err := time.Parse(layout, twitchFormat.CreatedAt) + if err != nil { + return ret, err + } + ret.Date = d + + var DuetRegex = regexp.MustCompile(`^Duet with ([^ ]*): (.*)$`) + matches := DuetRegex.FindAllStringSubmatch(twitchFormat.Title, -1) + if matches == nil { + var SoloRegex = regexp.MustCompile(`^Solo performance: (.*)$`) + matches := SoloRegex.FindAllStringSubmatch(twitchFormat.Title, -1) + if matches == nil { + ret.SongTitle = "Does not match either solo or duet performance regex " + ret.FullTitle = twitchFormat.Title + return ret, errors.New("Does not match either solo or duet performance regex : " + twitchFormat.Title) + } + ret.Duet = false + ret.SongTitle = matches[0][1] + ret.FullTitle = twitchFormat.Title + return ret, nil + } + ret.Duet = true + ret.OtherSinger = matches[0][1] + ret.SongTitle = matches[0][2] + ret.FullTitle = twitchFormat.Title + return ret, nil +} + +type SongSings struct { + SongTitle string + Sings int +} +type SingerSings struct { + SongTitle string + Sings int +} + +func calculateTopNSongs(songCache []irc.SingsVideoStruct, howMany int) []SongSings { + songMap := map[string]int{} + for _, record := range songCache { + sings := songMap[record.SongTitle] + sings += 1 + songMap[record.SongTitle] = sings + } + slice := make([]SongSings, 0) + for key, value := range songMap { + ss := SongSings{key, value} + slice = append(slice, ss) + } + sort.SliceStable(slice, func(i, j int) bool { + return slice[i].Sings > slice[j].Sings + }) + var ret []SongSings + for i := 1; i <= howMany; i++ { + ret = append(ret, slice[i]) + } + return ret +} + +func calculateLastSungSongDate(songCache []irc.SingsVideoStruct, SongTitle string) time.Time { + var t time.Time + for _, record := range songCache { + if record.SongTitle == SongTitle { + if record.Date.After(t) { + t = record.Date + } + } + } + return t +} + +func calculateLastSungSingerDate(songCache []irc.SingsVideoStruct, Singer string) time.Time { + var t time.Time + for _, record := range songCache { + if record.OtherSinger == Singer { + if record.Date.After(t) { + t = record.Date + } + } + } + return t +} + +func updateCalculatedFields(songCache []irc.SingsVideoStruct) { + for i, record := range songCache { + if record.Duet { + songCache[i].LastSungSinger = calculateLastSungSingerDate(songCache, record.OtherSinger) + } + songCache[i].LastSungSong = calculateLastSungSongDate(songCache, record.SongTitle) + } +} + +func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.SingsVideoStruct, error) { + url := "" + if from == "" { + url = "videos?user_id=" + userID + "&first=100&type=upload" + } else { + url = "videos?user_id=" + userID + "&first=100&type=upload&after=" + from + } + + vidResponse, err := twitchHTTPClient(url, bearer) + if err != nil { + return nil, err + } + + var fullResponse videosResponse + err = json.Unmarshal([]byte(vidResponse), &fullResponse) + if err != nil { + return nil, err + } + titles := make([]irc.SingsVideoStruct, 0) + for _, videoData := range fullResponse.Data { + ret, err := twitchVidToSingsVid(videoData) + if err != nil { + titles = append(titles, ret) + } else { + titles = append(titles, ret) + } + } + if fullResponse.Pagination.Cursor != "" { + fmt.Println("NOTICE: Recursion needed, cursor is " + fullResponse.Pagination.Cursor) + recurse, err := fetchVoDsPagesRecursive(userID, bearer, fullResponse.Pagination.Cursor) + if err != nil { + fmt.Println("ERROR: Bailing out during recursion because of error : " + err.Error()) + } + titles = append(titles, recurse...) + } + return titles, nil +} + +func fetchAllVoDs(userID string, bearer string) ([]irc.SingsVideoStruct, error) { + tokenValid, err := ValidateTwitchBearerToken(bearer) + if err != nil { + return nil, err + } + if !tokenValid { + return nil, errors.New("Failed to validate token with twitch (authorization revoked?!)") + } + titles, err := fetchVoDsPagesRecursive(userID, bearer, "") + if err != nil { + return make([]irc.SingsVideoStruct, 0), err + } + return titles, nil +} + func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) @@ -266,8 +478,23 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) { } user := usersObject.Data[0] - magicCode := ircBot.ReadOrCreateChannelKey(user.Login) + ircBot.UpdateBearerToken(user.Login, oauthResponse.Access_token) + if time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours() > 1 { + fmt.Printf("Cache of %d performances is older than an hour - %.1f hours old to be precise... fetching.\n", len(ircBot.ChannelData[user.Login].VideoCache), time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours()) + vids, err := fetchAllVoDs(user.Id, oauthResponse.Access_token) + if err != nil { + errCache := make([]irc.SingsVideoStruct, 0) + var ret irc.SingsVideoStruct + ret.FullTitle = "Error fetching videos: " + err.Error() + errCache = append(errCache, ret) + vids = errCache + } + updateCalculatedFields(vids) + ircBot.UpdateVideoCache(user.Login, vids) + } else { + fmt.Printf("Cache of %d performances is younger than an hour - %.1f hours old to be precise... not fetching.\n", len(ircBot.ChannelData[user.Login].VideoCache), time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours()) + } url := "https://" + ircBot.Config.ExternalUrl + "/admin/" + user.Login + "/" + magicCode http.Redirect(response, request, url, http.StatusFound) @@ -275,7 +502,7 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) { fmt.Fprintf(response, "I'm not okay jack! %v \n", vars) for key, val := range vars { - fmt.Fprint(response, "%s = %s\n", key, val) + fmt.Fprintf(response, "%s = %s\n", key, val) } } } diff --git a/web/admin.html b/web/admin.html index 1586723..9ca1882 100755 --- a/web/admin.html +++ b/web/admin.html @@ -65,6 +65,10 @@ .visibleSave { display: inline; } + th { + font-weight: bold; + font-style: italic; + } @@ -105,12 +109,47 @@ }

Karaokards admin panel for {{.Channel}}!!!

-
+ +
+ Top 10 Songs : + + + {{ range .TopNSongs }} + + {{ end }} +
WhatSings
{{ .SongTitle }}{{ .Sings }}
+
+ + +
+ -