Working but too slow

Signed-off-by: Martyn Ranyard <m@rtyn.berlin>
This commit is contained in:
Martyn 2020-07-15 21:19:06 +02:00
parent b891b74d4b
commit 4572f7dae1
8 changed files with 337 additions and 20 deletions

View File

@ -4,6 +4,9 @@ LDFLAGS=-ldflags "-X main.buildDate=${BUILD}"
.PHONY: build deps static
test:
go test git.martyn.berlin/martyn/twitchsingstools/internal/webserver
build:
go build ${LDFLAGS}

View File

@ -14,8 +14,8 @@ steps:
- mkdir -p /go/src/git.martyn.berlin/martyn
- ln -s /drone/src /go/src/git.martyn.berlin/martyn/twitchsingstools
- cd /go/src/git.martyn.berlin/martyn/twitchsingstools
- go get
- go build
- make test
- make
- name: publish
image: plugins/docker:18
@ -54,8 +54,8 @@ steps:
- mkdir -p /go/src/git.martyn.berlin/martyn
- ln -s /drone/src /go/src/git.martyn.berlin/martyn/twitchsingstools
- cd /go/src/git.martyn.berlin/martyn/twitchsingstools
- go get
- go build
- make test
- make
trigger:
ref:

View File

@ -29,6 +29,9 @@ spec:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
env:
- name: TSTOOLS_DATA_FOLDER
value: /data
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default "latest" }}"

View File

@ -51,6 +51,7 @@ ingress:
enabled: true
annotations:
kubernetes.io/ingress.class: nginx
cert-manager.io/cluster-issuer: letsencrypt
hosts:
- host: twitchsingstools.ing.martyn.berlin
paths:

View File

@ -85,14 +85,27 @@ type KardBot struct {
Config ConfigStruct
}
type SingsVideoStruct struct {
Date time.Time `json:"date"` // Golang date of creation
FullTitle string `json:"fullTitle"` // Full Title
Duet bool `json:"duet"` // Is it a duet?
OtherSinger string `json:"otherSinger"` // Twitch NAME of the other singer, extracted from the title
SongTitle string `json:"songTitle"` // extracted from title
LastSungSong time.Time `json:"LastSungSong"` // Last time this SONG was sung
LastSungSinger time.Time `json:"LastSungSinger"` // Last time a duet was sung with this SINGER, regardless of song, only Duets have this date initialised
}
type ChannelData struct {
Name string `json:"name"`
AdminKey string `json:"value,omitempty"`
Command string `json:"customcommand,omitempty"`
ExtraStrings string `json:"extrastrings,omitempty"`
JoinTime time.Time `json:"jointime"`
ControlChannel bool
HasLeft bool `json:"hasleft"`
Name string `json:"name"`
AdminKey string `json:"value,omitempty"`
Command string `json:"customcommand,omitempty"`
ExtraStrings string `json:"extrastrings,omitempty"`
JoinTime time.Time `json:"jointime"`
ControlChannel bool
HasLeft bool `json:"hasleft"`
VideoCache []SingsVideoStruct `json:"videoCache"`
VideoCacheUpdated time.Time `json:"videoCacheUpdated"`
Bearer string `json:"bearer"`
}
// Connects the bot to the Twitch IRC server. The bot will continue to try to connect until it
@ -130,6 +143,22 @@ func (bb *KardBot) ActiveChannels() int {
return count
}
func (bb *KardBot) UpdateVideoCache(user string, videos []SingsVideoStruct) {
record := bb.ChannelData[user]
fmt.Printf("Replacing cache of %d performances with a new cache of %d performances", len(record.VideoCache), len(videos))
record.VideoCache = videos
record.VideoCacheUpdated = time.Now()
bb.Database.Write("channelData", user, record)
bb.ChannelData[user] = record
}
func (bb *KardBot) UpdateBearerToken(user string, token string) {
record := bb.ChannelData[user]
record.Bearer = token
bb.Database.Write("channelData", user, record)
bb.ChannelData[user] = record
}
// Listens for and logs messages from chat. Responds to commands from the channel owner. The bot
// continues until it gets disconnected, told to shutdown, or forcefully shutdown.
func (bb *KardBot) HandleChat() error {

View File

@ -1,9 +1,13 @@
package webserver
import (
"errors"
"math/rand"
"regexp"
"sort"
irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc"
"github.com/dustin/go-humanize"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
@ -44,6 +48,31 @@ type twitchUsersBigResponse struct {
Data []twitchUser `json:"data"`
}
type twitchCursor struct {
Cursor string `json:"cursor"`
}
type videoStruct struct {
CreatedAt string `json:"created_at"` // Date when the video was created.
Description string `json:"description"` // Description of the video.
Duration string `json:"duration"` // Length of the video.
ID string `json:"id"` // ID of the video.
Language string `json:"language"` // Language of the video.
Pagination string `json:"pagination"` // A cursor value, to be used in a subsequent request to specify the starting point of the next set of results.
PublishedAt string `json:"published_at"` // Date when the video was published.
ThumbnailURL string `json:"thumbnail_url"` // Template URL for the thumbnail of the video.
Title string `json:"title"` // Title of the video.
Type string `json:"type"` // Type of video. Valid values: "upload", "archive", "highlight".
URL string `json:"url"` // URL of the video.
UserID string `json:"user_id"` // ID of the user who owns the video.
UserName string `json:"user_name"` // Display name corresponding to user_id.
ViewCount int `json:"view_count"` // Number of times the video has been viewed.
Viewable string `json:"viewable"` // Indicates whether the video is publicly viewable. Valid values: "public", "private".
}
type videosResponse struct {
Data []videoStruct `json:"data"`
Pagination twitchCursor `json:"pagination"`
}
var ircBot *irc.KardBot
func HealthHandler(response http.ResponseWriter, request *http.Request) {
@ -115,6 +144,13 @@ func LeaveHandler(response http.ResponseWriter, request *http.Request) {
TemplateHandler(response, request)
}
func humanTimeFromTimeString(s string) string {
return s
// format := "2006-01-02 15:04:05 +0000 UTC"
// d, _ := time.Parse(format, s)
// return humanize.Time(d)
}
func AdminHandler(response http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
@ -129,9 +165,12 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) {
SinceTimeUTC string
Leaving bool
HasLeft bool
SongData []irc.SingsVideoStruct
TopNSongs []SongSings
}
channelData := ircBot.ChannelData[vars["channel"]]
var td = TemplateData{channelData.Name, channelData.Command, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft}
topNSongs := calculateTopNSongs(channelData.VideoCache, 10)
var td = TemplateData{channelData.Name, channelData.Command, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft, channelData.VideoCache, topNSongs}
if request.Method == "POST" {
request.ParseForm()
@ -157,7 +196,7 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) {
}
ircBot.Database.Write("channelData", vars["channel"], record)
ircBot.ChannelData[vars["channel"]] = record
td = TemplateData{record.Name, record.Command, record.ExtraStrings, record.JoinTime, record.JoinTime.Format(irc.UTCFormat), false, record.HasLeft}
td = TemplateData{record.Name, record.Command, record.ExtraStrings, record.JoinTime, record.JoinTime.Format(irc.UTCFormat), false, record.HasLeft, channelData.VideoCache, topNSongs}
ircBot.JoinChannel(record.Name)
}
sourceData := ircBot.ChannelData[vars["channel"]]
@ -173,7 +212,12 @@ func AdminHandler(response http.ResponseWriter, request *http.Request) {
}
ircBot.Database.Write("channelData", vars["channel"], sourceData)
}
tmpl := template.Must(template.ParseFiles("web/admin.html"))
tmpl, err := template.New("admin.html").Funcs(template.FuncMap{
"niceDate": humanize.Time,
}).ParseFiles("web/admin.html")
if err != nil {
panic(err.Error())
}
tmpl.Execute(response, td)
}
@ -203,6 +247,174 @@ func twitchHTTPClient(call string, bearer string) (string, error) {
return string([]byte(body)), nil
}
func ValidateTwitchBearerToken(bearer string) (bool, error) {
url := "https://id.twitch.tv/oauth2/validate"
var bearerHeader = "Bearer " + bearer
req, err := http.NewRequest("GET", url, nil)
req.Header.Add("Client-ID", ircBot.AppCredentials.ClientID)
req.Header.Add("Authorization", bearerHeader)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return false, err
}
defer resp.Body.Close()
_, _ = ioutil.ReadAll(resp.Body)
return resp.StatusCode == 200, nil
}
func twitchVidToSingsVid(twitchFormat videoStruct) (irc.SingsVideoStruct, error) {
var ret irc.SingsVideoStruct
layout := "2006-01-02T15:04:05Z"
var d time.Time
d, err := time.Parse(layout, twitchFormat.CreatedAt)
if err != nil {
return ret, err
}
ret.Date = d
var DuetRegex = regexp.MustCompile(`^Duet with ([^ ]*): (.*)$`)
matches := DuetRegex.FindAllStringSubmatch(twitchFormat.Title, -1)
if matches == nil {
var SoloRegex = regexp.MustCompile(`^Solo performance: (.*)$`)
matches := SoloRegex.FindAllStringSubmatch(twitchFormat.Title, -1)
if matches == nil {
ret.SongTitle = "Does not match either solo or duet performance regex "
ret.FullTitle = twitchFormat.Title
return ret, errors.New("Does not match either solo or duet performance regex : " + twitchFormat.Title)
}
ret.Duet = false
ret.SongTitle = matches[0][1]
ret.FullTitle = twitchFormat.Title
return ret, nil
}
ret.Duet = true
ret.OtherSinger = matches[0][1]
ret.SongTitle = matches[0][2]
ret.FullTitle = twitchFormat.Title
return ret, nil
}
type SongSings struct {
SongTitle string
Sings int
}
type SingerSings struct {
SongTitle string
Sings int
}
func calculateTopNSongs(songCache []irc.SingsVideoStruct, howMany int) []SongSings {
songMap := map[string]int{}
for _, record := range songCache {
sings := songMap[record.SongTitle]
sings += 1
songMap[record.SongTitle] = sings
}
slice := make([]SongSings, 0)
for key, value := range songMap {
ss := SongSings{key, value}
slice = append(slice, ss)
}
sort.SliceStable(slice, func(i, j int) bool {
return slice[i].Sings > slice[j].Sings
})
var ret []SongSings
for i := 1; i <= howMany; i++ {
ret = append(ret, slice[i])
}
return ret
}
func calculateLastSungSongDate(songCache []irc.SingsVideoStruct, SongTitle string) time.Time {
var t time.Time
for _, record := range songCache {
if record.SongTitle == SongTitle {
if record.Date.After(t) {
t = record.Date
}
}
}
return t
}
func calculateLastSungSingerDate(songCache []irc.SingsVideoStruct, Singer string) time.Time {
var t time.Time
for _, record := range songCache {
if record.OtherSinger == Singer {
if record.Date.After(t) {
t = record.Date
}
}
}
return t
}
func updateCalculatedFields(songCache []irc.SingsVideoStruct) {
for i, record := range songCache {
if record.Duet {
songCache[i].LastSungSinger = calculateLastSungSingerDate(songCache, record.OtherSinger)
}
songCache[i].LastSungSong = calculateLastSungSongDate(songCache, record.SongTitle)
}
}
func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.SingsVideoStruct, error) {
url := ""
if from == "" {
url = "videos?user_id=" + userID + "&first=100&type=upload"
} else {
url = "videos?user_id=" + userID + "&first=100&type=upload&after=" + from
}
vidResponse, err := twitchHTTPClient(url, bearer)
if err != nil {
return nil, err
}
var fullResponse videosResponse
err = json.Unmarshal([]byte(vidResponse), &fullResponse)
if err != nil {
return nil, err
}
titles := make([]irc.SingsVideoStruct, 0)
for _, videoData := range fullResponse.Data {
ret, err := twitchVidToSingsVid(videoData)
if err != nil {
titles = append(titles, ret)
} else {
titles = append(titles, ret)
}
}
if fullResponse.Pagination.Cursor != "" {
fmt.Println("NOTICE: Recursion needed, cursor is " + fullResponse.Pagination.Cursor)
recurse, err := fetchVoDsPagesRecursive(userID, bearer, fullResponse.Pagination.Cursor)
if err != nil {
fmt.Println("ERROR: Bailing out during recursion because of error : " + err.Error())
}
titles = append(titles, recurse...)
}
return titles, nil
}
func fetchAllVoDs(userID string, bearer string) ([]irc.SingsVideoStruct, error) {
tokenValid, err := ValidateTwitchBearerToken(bearer)
if err != nil {
return nil, err
}
if !tokenValid {
return nil, errors.New("Failed to validate token with twitch (authorization revoked?!)")
}
titles, err := fetchVoDsPagesRecursive(userID, bearer, "")
if err != nil {
return make([]irc.SingsVideoStruct, 0), err
}
return titles, nil
}
func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
@ -266,8 +478,23 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
}
user := usersObject.Data[0]
magicCode := ircBot.ReadOrCreateChannelKey(user.Login)
ircBot.UpdateBearerToken(user.Login, oauthResponse.Access_token)
if time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours() > 1 {
fmt.Printf("Cache of %d performances is older than an hour - %.1f hours old to be precise... fetching.\n", len(ircBot.ChannelData[user.Login].VideoCache), time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours())
vids, err := fetchAllVoDs(user.Id, oauthResponse.Access_token)
if err != nil {
errCache := make([]irc.SingsVideoStruct, 0)
var ret irc.SingsVideoStruct
ret.FullTitle = "Error fetching videos: " + err.Error()
errCache = append(errCache, ret)
vids = errCache
}
updateCalculatedFields(vids)
ircBot.UpdateVideoCache(user.Login, vids)
} else {
fmt.Printf("Cache of %d performances is younger than an hour - %.1f hours old to be precise... not fetching.\n", len(ircBot.ChannelData[user.Login].VideoCache), time.Now().Sub(ircBot.ChannelData[user.Login].VideoCacheUpdated).Hours())
}
url := "https://" + ircBot.Config.ExternalUrl + "/admin/" + user.Login + "/" + magicCode
http.Redirect(response, request, url, http.StatusFound)
@ -275,7 +502,7 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
fmt.Fprintf(response, "I'm not okay jack! %v \n", vars)
for key, val := range vars {
fmt.Fprint(response, "%s = %s\n", key, val)
fmt.Fprintf(response, "%s = %s\n", key, val)
}
}
}

View File

@ -65,6 +65,10 @@
.visibleSave {
display: inline;
}
th {
font-weight: bold;
font-style: italic;
}
</style>
</head>
@ -105,12 +109,47 @@
}
</script>
<h1 class="cover-heading">Karaokards admin panel for {{.Channel}}!!!</h1>
<form method="POST">
<ul class="nav nav-tabs" style="width: 80%;">
<li class="nav-item">
<a id="insights" class="nav-link active panel" href="#" onclick="tabClick(this)">Insights</a>
</li>
<li class="nav-item">
<a id="data" class="nav-link panel" href="#" onclick="tabClick(this)">Data</a>
</li>
<li class="nav-item">
<a id="csv" class="nav-link panel" href="#" onclick="tabClick(this)">CSV</a>
</li>
<li class="nav-item">
<a id="bot" class="nav-link panel" href="#" onclick="tabClick(this)">Bot</a>
</li>
</ul>
<div style="width: 80%;overflow-y: scroll; display: block;" id="insightspanel" class="controlpanel">
Top 10 Songs :
<table>
<thead><tr><td>What</td><td>Sings</td>
{{ range .TopNSongs }}
<tr><td>{{ .SongTitle }}</td><td>{{ .Sings }}</td></tr>
{{ end }}
</table>
</div>
<div style="width: 80%;overflow-y: scroll; display: none;" id="datapanel" class="controlpanel">
<table>
<thead><tr><td>Published</td><td>Who</td><td>What</td><td>Last sang this song</td><td>Last dueted with performer</td></tr></thead>
{{ range .SongData }}
<tr><td title="{{ .Date }}">{{ .Date | niceDate }}</td><td>{{ .SongTitle }}</td><td>{{ .OtherSinger }}</td><td title="{{ .LastSungSong }}">{{ .LastSungSong | niceDate }}</td><td title="{{ .LastSungSinger }}">{{ .LastSungSinger | niceDate }}</td></tr>
{{ end }}
</table>
</div>
<div style="overflow-y: scroll; display: none;" id="csvpanel" class="controlpanel">
<h2>CSV support coming!</h2>
</div>
<div style="overflow-y: scroll; display: none;" id="botpanel" class="controlpanel">
{{ if .HasLeft }}
<h2>Not in your channel at the moment!</h2>
<p>The bot is not currently in your channel, chances are you've not ever asked it to join, you asked it to leave, or something went horribly wrong.</p>
<p>You can invite the bot to your channel by clicking here : <input id="joinButton" type="submit" name="join" value="Come on in"></p>
{{ else }}
<form method="POST">
{{ if .Leaving }}
<h2>Do you really want this bot to leave your channel?</h2>
<p><input id="leaveButton" type="submit" name="reallyleave" value="Really leave twitch channel"></p>
@ -129,9 +168,24 @@
</table>
{{ end }}
{{ end }}
</div>
</form>
<script type="text/javascript">
function tabClick(self) {
var theTabs = document.querySelectorAll(".panel")
for (tab of theTabs) {
tab.className = "nav-link panel"
}
self.className = "nav-link panel active"
var thePanels = document.querySelectorAll(".controlpanel")
for (panel of thePanels) {
panel.style.display = "none"
}
document.getElementById(self.id+"panel").style.display = "block"
}
</script>
</main>
<footer class="mastfoot mt-auto">
<footer class="mastfoot mt-auto">
<div class="inner">
<p>Cover template for <a href="https://getbootstrap.com/">Bootstrap</a>, by <a href="https://twitter.com/mdo">@mdo</a>.</p>
</div>
@ -140,4 +194,4 @@
</body>
</html>
</body>
</html>
</html>

View File

@ -55,7 +55,7 @@
<header class="masthead mb-auto">
<div class="inner">
<h3 class="masthead-brand">Twitch Sings Tools</h3>
<p>(Not endorsed by Twitch, the app is called "Twitch Sings" and these are tools for it, whaddya want it to be called?!)</p>
<p class="masthead-brand">(Not endorsed by Twitch, the app is called "Twitch Sings" and these are tools for it, whaddya want it to be called?!)</p>
<nav class="nav nav-masthead justify-content-center">
<a class="nav-link active" href="/">Home</a>
<a class="nav-link active" href="https://id.twitch.tv/oauth2/authorize?client_id={{.ClientID}}&redirect_uri={{.BaseURI}}/twitchadmin&response_type=code&scope=user:read:broadcast">Admin - log in with twitch</a>