Working but too slow

Signed-off-by: Martyn Ranyard <m@rtyn.berlin>

Makefile

@@ -4,6 +4,9 @@ LDFLAGS=-ldflags "-X main.buildDate=${BUILD}"
 
 .PHONY: build deps static
 
+test:
+	go test git.martyn.berlin/martyn/twitchsingstools/internal/webserver
+
 build:
 	go build ${LDFLAGS}
 

build/ci/drone.yml

@@ -14,8 +14,8 @@ steps:
   - mkdir -p /go/src/git.martyn.berlin/martyn
   - ln -s /drone/src /go/src/git.martyn.berlin/martyn/twitchsingstools
   - cd /go/src/git.martyn.berlin/martyn/twitchsingstools
-  - go get
+  - make test
-  - go build
+  - make
 
 - name: publish
   image: plugins/docker:18
@@ -54,8 +54,8 @@ steps:
   - mkdir -p /go/src/git.martyn.berlin/martyn
   - ln -s /drone/src /go/src/git.martyn.berlin/martyn/twitchsingstools
   - cd /go/src/git.martyn.berlin/martyn/twitchsingstools
-  - go get
+  - make test
-  - go build
+  - make
 
 trigger:
   ref:

deployments/helm/twitchsingstools/templates/deployment.yaml

@@ -29,6 +29,9 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
         - name: {{ .Chart.Name }}
+          env:
+          - name: TSTOOLS_DATA_FOLDER
+            value: /data
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default "latest" }}"

deployments/helm/twitchsingstools/values.yaml

@@ -51,6 +51,7 @@ ingress:
   enabled: true
   annotations: 
     kubernetes.io/ingress.class: nginx
+    cert-manager.io/cluster-issuer: letsencrypt
   hosts:
     - host: twitchsingstools.ing.martyn.berlin
       paths: 

internal/irc/irc.go

@@ -85,14 +85,27 @@ type KardBot struct {
 	Config         ConfigStruct
 }
 
+type SingsVideoStruct struct {
+	Date           time.Time `json:"date"`           // Golang date of creation
+	FullTitle      string    `json:"fullTitle"`      // Full Title
+	Duet           bool      `json:"duet"`           // Is it a duet?
+	OtherSinger    string    `json:"otherSinger"`    // Twitch NAME of the other singer, extracted from the title
+	SongTitle      string    `json:"songTitle"`      // extracted from title
+	LastSungSong   time.Time `json:"LastSungSong"`   // Last time this SONG was sung
+	LastSungSinger time.Time `json:"LastSungSinger"` // Last time a duet was sung with this SINGER, regardless of song, only Duets have this date initialised
+}
+
 type ChannelData struct {
-	Name           string    `json:"name"`
+	Name              string    `json:"name"`
-	AdminKey       string    `json:"value,omitempty"`
+	AdminKey          string    `json:"value,omitempty"`
-	Command        string    `json:"customcommand,omitempty"`
+	Command           string    `json:"customcommand,omitempty"`
-	ExtraStrings   string    `json:"extrastrings,omitempty"`
+	ExtraStrings      string    `json:"extrastrings,omitempty"`
-	JoinTime       time.Time `json:"jointime"`
+	JoinTime          time.Time `json:"jointime"`
-	ControlChannel bool
+	ControlChannel    bool
-	HasLeft        bool `json:"hasleft"`
+	HasLeft           bool               `json:"hasleft"`
+	VideoCache        []SingsVideoStruct `json:"videoCache"`
+	VideoCacheUpdated time.Time          `json:"videoCacheUpdated"`
+	Bearer            string             `json:"bearer"`
 }
 
 // Connects the bot to the Twitch IRC server. The bot will continue to try to connect until it
@@ -130,6 +143,22 @@ func (bb *KardBot) ActiveChannels() int {
 	return count
 }
 
+func (bb *KardBot) UpdateVideoCache(user string, videos []SingsVideoStruct) {
+	record := bb.ChannelData[user]
+	fmt.Printf("Replacing cache of %d performances with a new cache of %d performances", len(record.VideoCache), len(videos))
+	record.VideoCache = videos
+	record.VideoCacheUpdated = time.Now()
+	bb.Database.Write("channelData", user, record)
+	bb.ChannelData[user] = record
+}
+
+func (bb *KardBot) UpdateBearerToken(user string, token string) {
+	record := bb.ChannelData[user]
+	record.Bearer = token
+	bb.Database.Write("channelData", user, record)
+	bb.ChannelData[user] = record
+}
+
 // Listens for and logs messages from chat. Responds to commands from the channel owner. The bot
 // continues until it gets disconnected, told to shutdown, or forcefully shutdown.
 func (bb *KardBot) HandleChat() error {

internal/webserver/webserver.go

@@ -1,9 +1,13 @@
 package webserver
 
 import (
+	"errors"
 	"math/rand"
+	"regexp"
+	"sort"
 
 	irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc"
+	"github.com/dustin/go-humanize"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 
@@ -44,6 +48,31 @@ type twitchUsersBigResponse struct {
 	Data []twitchUser `json:"data"`
 }
 
+type twitchCursor struct {
+	Cursor string `json:"cursor"`
+}
+type videoStruct struct {
+	CreatedAt    string `json:"created_at"`    // Date when the video was created.
+	Description  string `json:"description"`   // Description of the video.
+	Duration     string `json:"duration"`      // Length of the video.
+	ID           string `json:"id"`            // ID of the video.
+	Language     string `json:"language"`      // Language of the video.
+	Pagination   string `json:"pagination"`    // A cursor value, to be used in a subsequent request to specify the starting point of the next set of results.
+	PublishedAt  string `json:"published_at"`  // Date when the video was published.
+	ThumbnailURL string `json:"thumbnail_url"` // Template URL for the thumbnail of the video.
+	Title        string `json:"title"`         // Title of the video.
+	Type         string `json:"type"`          // Type of video. Valid values: "upload", "archive", "highlight".
+	URL          string `json:"url"`           // URL of the video.
+	UserID       string `json:"user_id"`       // ID of the user who owns the video.
+	UserName     string `json:"user_name"`     // Display name corresponding to user_id.
+	ViewCount    int    `json:"view_count"`    // Number of times the video has been viewed.
+	Viewable     string `json:"viewable"`      // Indicates whether the video is publicly viewable. Valid values: "public", "private".
+}
+type videosResponse struct {
+	Data       []videoStruct `json:"data"`
+	Pagination twitchCursor  `json:"pagination"`
+}
+
 var ircBot *irc.KardBot
 
 func HealthHandler(response http.ResponseWriter, request *http.Request) {
@@ -115,6 +144,13 @@ func LeaveHandler(response http.ResponseWriter, request *http.Request) {
 	TemplateHandler(response, request)
 }
 
+func humanTimeFromTimeString(s string) string {
+	return s
+	// format := "2006-01-02 15:04:05 +0000 UTC"
+	// d, _ := time.Parse(format, s)
+	// return humanize.Time(d)
+}
+
 func AdminHandler(response http.ResponseWriter, request *http.Request) {
 	vars := mux.Vars(request)
 	if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
@@ -129,9 +165,12 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) {
 		SinceTimeUTC string
 		Leaving      bool
 		HasLeft      bool
+		SongData     []irc.SingsVideoStruct
+		TopNSongs    []SongSings
 	}
 	channelData := ircBot.ChannelData[vars["channel"]]
-	var td = TemplateData{channelData.Name, channelData.Command, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft}
+	topNSongs := calculateTopNSongs(channelData.VideoCache, 10)
+	var td = TemplateData{channelData.Name, channelData.Command, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft, channelData.VideoCache, topNSongs}
 
 	if request.Method == "POST" {
 		request.ParseForm()
@@ -157,7 +196,7 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) {
 			}
 			ircBot.Database.Write("channelData", vars["channel"], record)
 			ircBot.ChannelData[vars["channel"]] = record
-			td = TemplateData{record.Name, record.Command, record.ExtraStrings, record.JoinTime, record.JoinTime.Format(irc.UTCFormat), false, record.HasLeft}
+			td = TemplateData{record.Name, record.Command, record.ExtraStrings, record.JoinTime, record.JoinTime.Format(irc.UTCFormat), false, record.HasLeft, channelData.VideoCache, topNSongs}
 			ircBot.JoinChannel(record.Name)
 		}
 		sourceData := ircBot.ChannelData[vars["channel"]]
@@ -173,7 +212,12 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) {
 		}
 		ircBot.Database.Write("channelData", vars["channel"], sourceData)
 	}
-	tmpl := template.Must(template.ParseFiles("web/admin.html"))
+	tmpl, err := template.New("admin.html").Funcs(template.FuncMap{
+		"niceDate": humanize.Time,
+	}).ParseFiles("web/admin.html")
+	if err != nil {
+		panic(err.Error())
+	}
 	tmpl.Execute(response, td)
 }
 
@@ -203,6 +247,174 @@ func twitchHTTPClient(call string, bearer string) (string, error) {
 	return string([]byte(body)), nil
 }
 
+func ValidateTwitchBearerToken(bearer string) (bool, error) {
+	url := "https://id.twitch.tv/oauth2/validate"
+	var bearerHeader = "Bearer " + bearer
+
+	req, err := http.NewRequest("GET", url, nil)
+	req.Header.Add("Client-ID", ircBot.AppCredentials.ClientID)
+	req.Header.Add("Authorization", bearerHeader)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	_, _ = ioutil.ReadAll(resp.Body)
+	return resp.StatusCode == 200, nil
+}
+
+func twitchVidToSingsVid(twitchFormat videoStruct) (irc.SingsVideoStruct, error) {
+	var ret irc.SingsVideoStruct
+	layout := "2006-01-02T15:04:05Z"
+	var d time.Time
+	d, err := time.Parse(layout, twitchFormat.CreatedAt)
+	if err != nil {
+		return ret, err
+	}
+	ret.Date = d
+
+	var DuetRegex = regexp.MustCompile(`^Duet with ([^ ]*): (.*)$`)
+	matches := DuetRegex.FindAllStringSubmatch(twitchFormat.Title, -1)
+	if matches == nil {
+		var SoloRegex = regexp.MustCompile(`^Solo performance: (.*)$`)
+		matches := SoloRegex.FindAllStringSubmatch(twitchFormat.Title, -1)
+		if matches == nil {
+			ret.SongTitle = "Does not match either solo or duet performance regex "
+			ret.FullTitle = twitchFormat.Title
+			return ret, errors.New("Does not match either solo or duet performance regex : " + twitchFormat.Title)
+		}
+		ret.Duet = false
+		ret.SongTitle = matches[0][1]
+		ret.FullTitle = twitchFormat.Title
+		return ret, nil
+	}
+	ret.Duet = true
+	ret.OtherSinger = matches[0][1]
+	ret.SongTitle = matches[0][2]
+	ret.FullTitle = twitchFormat.Title
+	return ret, nil
+}
+
+type SongSings struct {
+	SongTitle string
+	Sings     int
+}
+type SingerSings struct {
+	SongTitle string
+	Sings     int
+}
+
+func calculateTopNSongs(songCache []irc.SingsVideoStruct, howMany int) []SongSings {
+	songMap := map[string]int{}
+	for _, record := range songCache {
+		sings := songMap[record.SongTitle]
+		sings += 1
+		songMap[record.SongTitle] = sings
+	}
+	slice := make([]SongSings, 0)
+	for key, value := range songMap {
+		ss := SongSings{key, value}
+		slice = append(slice, ss)
+	}
+	sort.SliceStable(slice, func(i, j int) bool {
+		return slice[i].Sings > slice[j].Sings
+	})
+	var ret []SongSings
+	for i := 1; i <= howMany; i++ {
+		ret = append(ret, slice[i])
+	}
+	return ret
+}
+
+func calculateLastSungSongDate(songCache []irc.SingsVideoStruct, SongTitle string) time.Time {
+	var t time.Time
+	for _, record := range songCache {
+		if record.SongTitle == SongTitle {
+			if record.Date.After(t) {
+				t = record.Date
+			}
+		}
+	}
+	return t
+}
+
+func calculateLastSungSingerDate(songCache []irc.SingsVideoStruct, Singer string) time.Time {
+	var t time.Time
+	for _, record := range songCache {
+		if record.OtherSinger == Singer {
+			if record.Date.After(t) {
+				t = record.Date
+			}
+		}
+	}
+	return t
+}
+
+func updateCalculatedFields(songCache []irc.SingsVideoStruct) {
+	for i, record := range songCache {
+		if record.Duet {
+			songCache[i].LastSungSinger = calculateLastSungSingerDate(songCache, record.OtherSinger)
+		}
+		songCache[i].LastSungSong = calculateLastSungSongDate(songCache, record.SongTitle)
+	}
+}
+
+func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.SingsVideoStruct, error) {
+	url := ""
+	if from == "" {
+		url = "videos?user_id=" + userID + "&first=100&type=upload"
+	} else {
+		url = "videos?user_id=" + userID + "&first=100&type=upload&after=" + from
+	}
+
+	vidResponse, err := twitchHTTPClient(url, bearer)
+	if err != nil {
+		return nil, err
+	}
+
+	var fullResponse videosResponse
+	err = json.Unmarshal([]byte(vidResponse), &fullResponse)
+	if err != nil {
+		return nil, err
+	}
+	titles := make([]irc.SingsVideoStruct, 0)
+	for _, videoData := range fullResponse.Data {
+		ret, err := twitchVidToSingsVid(videoData)
+		if err != nil {
+			titles = append(titles, ret)
+		} else {
+			titles = append(titles, ret)
+		}
+	}
+	if fullResponse.Pagination.Cursor != "" {
+		fmt.Println("NOTICE: Recursion needed, cursor is " + fullResponse.Pagination.Cursor)
+		recurse, err := fetchVoDsPagesRecursive(userID, bearer, fullResponse.Pagination.Cursor)
+		if err != nil {
+			fmt.Println("ERROR: Bailing out during recursion because of error : " + err.Error())
+		}
+		titles = append(titles, recurse...)
+	}
+	return titles, nil
+}
+
+func fetchAllVoDs(userID string, bearer string) ([]irc.SingsVideoStruct, error) {
+	tokenValid, err := ValidateTwitchBearerToken(bearer)
+	if err != nil {
+		return nil, err
+	}
+	if !tokenValid {
+		return nil, errors.New("Failed to validate token with twitch (authorization revoked?!)")
+	}
+	titles, err := fetchVoDsPagesRecursive(userID, bearer, "")
+	if err != nil {
+		return make([]irc.SingsVideoStruct, 0), err
+	}
+	return titles, nil
+}
+
 func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
 
 	vars := mux.Vars(request)
@@ -266,8 +478,23 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
 		}
 
 		user := usersObject.Data[0]
-
 		magicCode := ircBot.ReadOrCreateChannelKey(user.Login)
+		ircBot.UpdateBearerToken(user.Login, oauthResponse.Access_token)
+		if time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours() > 1 {
+			fmt.Printf("Cache of %d performances is older than an hour - %.1f hours old to be precise... fetching.\n", len(ircBot.ChannelData[user.Login].VideoCache), time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours())
+			vids, err := fetchAllVoDs(user.Id, oauthResponse.Access_token)
+			if err != nil {
+				errCache := make([]irc.SingsVideoStruct, 0)
+				var ret irc.SingsVideoStruct
+				ret.FullTitle = "Error fetching videos: " + err.Error()
+				errCache = append(errCache, ret)
+				vids = errCache
+			}
+			updateCalculatedFields(vids)
+			ircBot.UpdateVideoCache(user.Login, vids)
+		} else {
+			fmt.Printf("Cache of %d performances is younger than an hour - %.1f hours old to be precise... not fetching.\n", len(ircBot.ChannelData[user.Login].VideoCache), time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours())
+		}
 		url := "https://" + ircBot.Config.ExternalUrl + "/admin/" + user.Login + "/" + magicCode
 		http.Redirect(response, request, url, http.StatusFound)
 
@@ -275,7 +502,7 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
 
 		fmt.Fprintf(response, "I'm not okay jack! %v \n", vars)
 		for key, val := range vars {
-			fmt.Fprint(response, "%s = %s\n", key, val)
+			fmt.Fprintf(response, "%s = %s\n", key, val)
 		}
 	}
 }

web/admin.html

@@ -65,6 +65,10 @@
                     .visibleSave {
                       display: inline;
                     }
+                    th {
+                      font-weight: bold;
+                      font-style: italic;
+                    }
         
             </style>
           </head>
@@ -105,12 +109,47 @@
                     }
                     </script>
                         <h1 class="cover-heading">Karaokards admin panel for {{.Channel}}!!!</h1>
-                        <form method="POST">
+                        <ul class="nav nav-tabs" style="width: 80%;">
+                          <li class="nav-item">
+                            <a id="insights" class="nav-link active panel" href="#" onclick="tabClick(this)">Insights</a>
+                          </li>
+                          <li class="nav-item">
+                            <a id="data" class="nav-link panel" href="#" onclick="tabClick(this)">Data</a>
+                          </li>
+                          <li class="nav-item">
+                            <a id="csv" class="nav-link panel" href="#" onclick="tabClick(this)">CSV</a>
+                          </li>
+                          <li class="nav-item">
+                            <a id="bot" class="nav-link panel" href="#" onclick="tabClick(this)">Bot</a>
+                          </li>
+                        </ul>
+                        <div style="width: 80%;overflow-y: scroll; display: block;" id="insightspanel" class="controlpanel">
+                          Top 10 Songs : 
+                          <table>
+                            <thead><tr><td>What</td><td>Sings</td>
+                            {{ range .TopNSongs }}
+                            <tr><td>{{ .SongTitle }}</td><td>{{ .Sings }}</td></tr>
+                            {{ end }}
+                          </table>
+                        </div>
+                        <div style="width: 80%;overflow-y: scroll; display: none;" id="datapanel" class="controlpanel">
+                          <table>
+                            <thead><tr><td>Published</td><td>Who</td><td>What</td><td>Last sang this song</td><td>Last dueted with performer</td></tr></thead>
+                            {{ range .SongData }}
+                            <tr><td title="{{ .Date }}">{{ .Date | niceDate }}</td><td>{{ .SongTitle }}</td><td>{{ .OtherSinger }}</td><td title="{{ .LastSungSong }}">{{ .LastSungSong | niceDate }}</td><td title="{{ .LastSungSinger }}">{{ .LastSungSinger | niceDate }}</td></tr>
+                            {{ end }}
+                          </table>
+                        </div>
+                        <div style="overflow-y: scroll; display: none;" id="csvpanel" class="controlpanel">
+                          <h2>CSV support coming!</h2>
+                        </div>
+                        <div style="overflow-y: scroll; display: none;" id="botpanel" class="controlpanel">                          
                         {{ if .HasLeft }}
                         <h2>Not in your channel at the moment!</h2>
                         <p>The bot is not currently in your channel, chances are you've not ever asked it to join, you asked it to leave, or something went horribly wrong.</p>
                         <p>You can invite the bot to your channel by clicking here : <input id="joinButton" type="submit" name="join" value="Come on in"></p>
                         {{ else }}
+                          <form method="POST">
                           {{ if .Leaving }}
                           <h2>Do you really want this bot to leave your channel?</h2>
                           <p><input id="leaveButton" type="submit" name="reallyleave" value="Really leave twitch channel"></p>
@@ -129,9 +168,24 @@
                           </table>
                           {{ end }}
                         {{ end }}
+                        </div>
                         </form>
+                        <script type="text/javascript">
+                          function tabClick(self) {                  
+                            var theTabs = document.querySelectorAll(".panel")
+                            for (tab of theTabs) {
+                              tab.className = "nav-link panel"
+                            }
+                            self.className = "nav-link panel active"
+                            var thePanels = document.querySelectorAll(".controlpanel")
+                            for (panel of thePanels) {
+                              panel.style.display = "none"
+                            }
+                            document.getElementById(self.id+"panel").style.display = "block"
+                          }
+                        </script>
                 </main>
-        <footer class="mastfoot mt-auto">
+                <footer class="mastfoot mt-auto">
                 <div class="inner">
                   <p>Cover template for <a href="https://getbootstrap.com/">Bootstrap</a>, by <a href="https://twitter.com/mdo">@mdo</a>.</p>
                 </div>
@@ -140,4 +194,4 @@
                 </body>
                 </html>
     </body>
-</html>
+</html>

web/index.html

@@ -55,7 +55,7 @@
                 <header class="masthead mb-auto">
                 <div class="inner">
                   <h3 class="masthead-brand">Twitch Sings Tools</h3>
-                  <p>(Not endorsed by Twitch, the app is called "Twitch Sings" and these are tools for it, whaddya want it to be called?!)</p>
+                  <p class="masthead-brand">(Not endorsed by Twitch, the app is called "Twitch Sings" and these are tools for it, whaddya want it to be called?!)</p>
                   <nav class="nav nav-masthead justify-content-center">
                     <a class="nav-link active" href="/">Home</a>
                     <a class="nav-link active" href="https://id.twitch.tv/oauth2/authorize?client_id={{.ClientID}}&redirect_uri={{.BaseURI}}/twitchadmin&response_type=code&scope=user:read:broadcast">Admin - log in with twitch</a>