Add link to BockTown script

Signed-off-by: Martyn Ranyard <m@rtyn.berlin>
Redis refactor and video download script.
2020-09-08 21:25:59 +02:00 · 2020-09-05 18:14:56 +02:00 · 2020-08-10 12:10:58 +02:00
14 changed files with 603 additions and 252 deletions
--- a/build/react-frontend/src/Components/AppAdmin/AppAdmin.js
+++ b/build/react-frontend/src/Components/AppAdmin/AppAdmin.js
@ -100,6 +100,8 @@ class AppAdmin extends React.Component {
    const adminToken = window.location.href.split("/")[5]
    const csvURL = "/csv/"+ channelName + "/" + adminToken
    const tsvURL = "/tsv/"+ channelName + "/" + adminToken
+    const scriptURL = "/script.bat/"+ channelName + "/" + adminToken
+    const bockScriptURL = "https://tsdownloader.azurewebsites.net/api/ScriptApi/GenerateScript?userName="+ channelName
    const setPage = this.setPage
    const reloadDuetComponent = this.state.reloadDuetComponent
    const reloadSongComponent = this.state.reloadSongComponent
@ -121,9 +123,10 @@ class AppAdmin extends React.Component {
              <Tab label="Top Songs" {...a11yProps(0)}/>
              <Tab label="Top Singers" {...a11yProps(1)}/>
              <Tab label="Data" {...a11yProps(2)}/>
-              <Tab label="Export" {...a11yProps(3)}/>
-              <Tab label="Cache Details" {...a11yProps(4)}/>
-              <Tab label="Chatbot" {...a11yProps(4)}/>
+              <Tab label="Download Videos" {...a11yProps(3)}/>
+              <Tab label="Export" {...a11yProps(4)}/>
+              <Tab label="Cache Details" {...a11yProps(5)}/>
+              <Tab label="Chatbot" {...a11yProps(6)}/>
            </Tabs>
          </AppBar>
          <TabPanel value={page} index={0}>
@ -139,6 +142,32 @@ class AppAdmin extends React.Component {
             onReloadedChange={this.reloadDuetComponent}/>
          </TabPanel>
          <TabPanel value={page} index={3}>
+          <Typography variant="h3" component="h2" gutterBottom>
+            RIP Twitch sings!
+          </Typography>
+          <Typography variant="h5" component="h2" gutterBottom>
+            Here is a script to download all your published duets and solo performances.
+          </Typography>
+          <Typography variant="p">
+            First, you need to <a href="https://github.com/ytdl-org/youtube-dl/releases/download/2020.07.28/youtube-dl.exe">download youtube-dl from here</a>.
+            Then, download your <a href={scriptURL}>personal script from here</a>.<br/><br/>
+          </Typography>
+          <Typography variant="p">
+            Put both files in the same folder and run script.bat.  I know, that's not a nice interface, but, it works.  Right now, I can't bear to do much more, 
+            hurts to even think about the situation.  It creates a subfolder per person and dates the files.  
+            It even works if you sang the same song with the same person on the same day.<br/><br/>
+          </Typography>
+          <Typography variant="p">
+            If you need to pause, press "Control" and "C" and say "Y" to the question.  Next run the script will actually happily resume where it left off.<br/><br/>
+          </Typography>
+          <Typography variant="h5" component="h2" gutterBottom>
+            You might be interested in BockTown's script that does the other side of the coin - finds and downloads duets completed by others of your open seeds 
+          </Typography>
+          <Typography variant="p">
+            You can download your version of his script <a href={bockScriptURL}>from here</a>.  It'll take a bit to generate so be patient, click the link once and wait. ;-)
+          </Typography>
+          </TabPanel>
+          <TabPanel value={page} index={4}>
          <Typography variant="h5" component="h2" gutterBottom>
            Download your data as :
          </Typography>
--- a/deployments/helm/twitchsingstools/templates/deployment-db.yaml
+++ b/deployments/helm/twitchsingstools/templates/deployment-db.yaml
@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "twitchsingstools.fullname" . }}-db
+  annotations:
+    checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+    checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+  labels:
+    {{- include "twitchsingstools.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "twitchsingstools.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "twitchsingstools.selectorLabels" . | nindent 8 }}
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+    spec:
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      serviceAccountName: {{ include "twitchsingstools.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.db.image.repository }}:{{ .Values.db.image.tag | default "latest" }}"
+          imagePullPolicy: {{ .Values.db.image.pullPolicy }}
+          ports:
+            - name: redis
+              containerPort: 9221
+              protocol: TCP
+          livenessProbe:
+            tcpSocket:
+              port: redis
+          readinessProbe:
+            tcpSocket:
+              port: redis
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+          - mountPath: /pika/db
+            name: data
+      volumes:
+      - name: data
+        persistentVolumeClaim: 
+          claimName: tstools-db
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
--- a/deployments/helm/twitchsingstools/templates/deployment.yaml
+++ b/deployments/helm/twitchsingstools/templates/deployment.yaml
@ -32,6 +32,8 @@ spec:
          env:
          - name: TSTOOLS_DATA_FOLDER
            value: /data
+          - name: TSTOOLS_REDIS_HOST
+            value: {{ .Values.db.service.name }}
          securityContext:
            {{- toYaml .Values.securityContext | nindent 12 }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default "latest" }}"
--- a/deployments/helm/twitchsingstools/templates/pvc-db.yaml
+++ b/deployments/helm/twitchsingstools/templates/pvc-db.yaml
@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: tstools-db
+  labels:
+    {{- include "twitchsingstools.labels" . | nindent 4 }}
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.db.storageSize }}
--- a/deployments/helm/twitchsingstools/templates/service-db.yaml
+++ b/deployments/helm/twitchsingstools/templates/service-db.yaml
@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.db.service.name }}
+  labels:
+    {{- include "twitchsingstools.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.db.service.type }}
+  ports:
+    - port: {{ .Values.db.service.port }}
+      targetPort: redis
+      protocol: TCP
+      name: redis
+  selector:
+    {{- include "twitchsingstools.selectorLabels" . | nindent 4 }}
--- a/deployments/helm/twitchsingstools/values-dev.yaml
+++ b/deployments/helm/twitchsingstools/values-dev.yaml
@ -6,7 +6,7 @@ replicaCount: 1

 image:
  repository: imartyn/twitchsingstools
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
  tag: dev

 imagePullSecrets: []
--- a/deployments/helm/twitchsingstools/values-prod.yaml
+++ b/deployments/helm/twitchsingstools/values-prod.yaml
@ -0,0 +1,81 @@
+# Default values for twitchsingstools.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: imartyn/twitchsingstools
+  tag: 0.0-linux-amd64
+  pullPolicy: Always
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+secretFiles: {}
+
+irc:
+  nick: tstools
+
+twitchapp: {}
+
+storageSize: 10Gi
+
+service:
+  type: ClusterIP
+  port: 80
+
+externalHostname: twitchsingstools.martyn.berlin
+
+ingress:
+  enabled: true
+  annotations: 
+    kubernetes.io/ingress.class: traefik
+    cert-manager.io/cluster-issuer: letsencrypt
+  hosts:
+    - host: twitchsingstools.martyn.berlin
+      paths:
+      - /
+  tls: 
+    - secretName: tstools-tls
+      hosts:
+        - twitchsingstools.martyn.berlin
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
--- a/deployments/helm/twitchsingstools/values.yaml
+++ b/deployments/helm/twitchsingstools/values.yaml
@ -41,6 +41,15 @@ twitchapp: {}

 storageSize: 10Gi

+db:
+  service:
+    port: 4920
+    name: tstools-db
+  storageSize: 10Gi
+  image:
+    repository: pikadb/pika
+    pullPolicy: IfNotPresent
+
 service:
  type: ClusterIP
  port: 80
--- a/internal/data/data.go
+++ b/internal/data/data.go
@ -0,0 +1,234 @@
+package data
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"log"
+	"os"
+	"time"
+
+	rgb "github.com/foresthoffman/rgblog"
+	redis "github.com/gomodule/redigo/redis"
+	uuid "github.com/google/uuid"
+)
+
+// ConfigStruct is the base for the config file
+type ConfigStruct struct {
+	InitialChannels []string `json:"channels"`
+	IrcOAuthPath    string   `json:"ircoauthpath,omitempty"`
+	StringPath      string   `json:"authpath,omitempty"`
+	DataPath        string   `json:"datapath,omitempty"`
+	ExternalURL     string   `json:"externalurl,omitempty"`
+	AppOAuthPath    string   `json:"appoauthpath,omitempty"`
+	DatabaseSVC     string   `json:"databasesvc,omitempty"`
+}
+
+// CommandType Kinda an enum
+type CommandType string
+
+// CommandType literals
+const (
+	RandomSinger CommandType = "RandomSinger"
+	RandomPrompt CommandType = "RandomPrompt"
+	RandomSong   CommandType = "RandomSong"
+	AgingSinger  CommandType = "AgingSinger"
+	AgingSong    CommandType = "AgingSong"
+)
+
+// IsValid Is CommandType a valid enum?
+func (ct CommandType) IsValid() error {
+	switch ct {
+	case RandomSinger, RandomPrompt, RandomSong, AgingSinger, AgingSong:
+		return nil
+	}
+	return errors.New("Invalid command type")
+}
+
+// ChannelData is what we store in the BitRaft (Redis) database
+type ChannelData struct {
+	ControlChannel    bool
+	Name              string             `json:"name"`
+	AdminKey          string             `json:"value,omitempty"`
+	Commands          []CommandStruct    `json:"commands,omitempty"`
+	ExtraStrings      string             `json:"extrastrings,omitempty"`
+	JoinTime          time.Time          `json:"jointime"`
+	HasLeft           bool               `json:"hasleft"`
+	VideoCache        []SingsVideoStruct `json:"videoCache"`
+	VideoCacheUpdated time.Time          `json:"videoCacheUpdated"`
+	Bearer            string             `json:"bearer"`
+	TwitchUserID      string             `json:"twitchUserID"`
+}
+
+// SingsVideoStruct The data we pull from Twitch
+type SingsVideoStruct struct {
+	Date           time.Time `json:"date"`           // Golang date of creation
+	FullTitle      string    `json:"fullTitle"`      // Full Title
+	Duet           bool      `json:"duet"`           // Is it a duet?
+	OtherSinger    string    `json:"otherSinger"`    // Twitch NAME of the other singer, extracted from the title
+	SongTitle      string    `json:"songTitle"`      // extracted from title
+	LastSungSong   time.Time `json:"LastSungSong"`   // Last time this SONG was sung
+	LastSungSinger time.Time `json:"LastSungSinger"` // Last time a duet was sung with this SINGER, regardless of song, only Duets have this date initialised
+	VideoURL       string    `json:"VideoURL"`       // RIP Twitch Sings
+}
+
+// CommandStruct keypair for irc command -> actual thing to do
+type CommandStruct struct {
+	CommandName CommandType `json:"commandName"`
+	KeyWord     string      `json:"keyword"`
+}
+
+// GlobalData Some kind of architect would kill me for this
+type GlobalData struct {
+	ChannelData    map[string]ChannelData
+	Config         ConfigStruct
+	Database       redis.Conn
+	ControlChannel string
+}
+
+// ConnectDatabase Connects to the database set in the config struct
+func (gd *GlobalData) ConnectDatabase() {
+	var err error
+	rgb.YPrintf("[%s] Connecting to \"redis\" %s...\n", TimeStamp(), gd.Config.DatabaseSVC, err)
+	gd.Database, err = redis.Dial("tcp", gd.Config.DatabaseSVC+":4920")
+	if err != nil {
+		rgb.RPrintf("[%s] Failed connecting to \"redis\" %s : %s\n", TimeStamp(), gd.Config.DatabaseSVC, err)
+		os.Exit(1)
+	}
+	rgb.YPrintf("[%s] No error... wtf?\n", TimeStamp())
+}
+
+// UpdateVideoCache Updates the in-memory data and updates redis
+func (gd *GlobalData) UpdateVideoCache(user string, videos []SingsVideoStruct) {
+	record := gd.ChannelData[user]
+	rgb.YPrintf("Replacing cache of %d performances with a new cache of %d performances\n", len(record.VideoCache), len(videos))
+	record.VideoCache = videos
+	record.VideoCacheUpdated = time.Now()
+	asJson, _ := json.Marshal(record)
+	_, err := gd.Database.Do("SET", user, asJson)
+	if err != nil {
+		log.Fatal(err)
+	}
+	gd.ChannelData[user] = record
+}
+
+func (gd *GlobalData) UpdateBearerToken(user string, token string) {
+	record := gd.ChannelData[user]
+	record.Bearer = token
+	asJson, _ := json.Marshal(record)
+	gd.Database.Do("SET", user, asJson)
+	gd.ChannelData[user] = record
+}
+
+func (gd *GlobalData) UpdateTwitchUserID(user string, userid string) {
+	record := gd.ChannelData[user]
+	record.TwitchUserID = userid
+	asJson, _ := json.Marshal(record)
+	gd.Database.Do("SET", user, asJson)
+	gd.ChannelData[user] = record
+}
+
+func (gd *GlobalData) UpdateChannelKey(user string, channelKey string) {
+	record := gd.ChannelData[user]
+	record.AdminKey = channelKey
+	asJson, _ := json.Marshal(record)
+	gd.Database.Do("SET", user, asJson)
+	gd.ChannelData[user] = record
+}
+
+func (gd *GlobalData) UpdateChannelName(user string, newName string) {
+	record := gd.ChannelData[user]
+	record.Name = newName
+	asJson, _ := json.Marshal(record)
+	gd.Database.Do("SET", newName, asJson)
+	gd.ChannelData[newName] = record
+	//dunno why we'd need this but I guess in case?
+	if newName != user {
+		delete(gd.ChannelData, user)
+		gd.Database.Do("DEL", newName)
+	}
+}
+
+func (gd *GlobalData) UpdateJoined(user string, invert bool) {
+	record := gd.ChannelData[user]
+
+	if record.Name == "" {
+		record = ChannelData{Name: user, JoinTime: time.Now(), Commands: nil, ControlChannel: true}
+	}
+	record.JoinTime = time.Now()
+	asJson, _ := json.Marshal(record)
+	if invert {
+		record.HasLeft = true
+	} else {
+		record.HasLeft = false
+	}
+	gd.Database.Do("SET", user, asJson)
+	gd.ChannelData[user] = record
+}
+
+func (gd *GlobalData) ReadChannelData() error {
+	keys, err := redis.Strings(gd.Database.Do("KEYS", "*"))
+	if err != nil {
+		rgb.RPrintf("[%s] ERROR with redis fetch : %s\n", TimeStamp(), err.Error())
+		rgb.YPrintf("[%s] Maybe an empty redis, creating a record...\n", TimeStamp())
+		keys = []string{}
+	}
+	if len(keys) == 0 {
+		rgb.YPrintf("[%s] Looks like an empty redis, creating a record...\n", TimeStamp())
+		record := ChannelData{Name: gd.ControlChannel, JoinTime: time.Now(), Commands: nil}
+		asJSON, _ := json.Marshal(record)
+		gd.Database.Do("SET", gd.ControlChannel, asJSON)
+		keys = []string{gd.ControlChannel}
+	}
+	rgb.YPrintf("[%s] \"redis\" has %d records!\n", TimeStamp(), len(keys))
+	for _, channel := range keys {
+		fetchedData, err := redis.String(gd.Database.Do("GET", channel))
+		if err != nil {
+			rgb.YPrintf("[%s] failed to read key %s from redis, good luck!...\n", TimeStamp(), channel)
+		}
+		rgb.YPrintf("[%s] data from \"redis\" for %s is %s\n", TimeStamp(), channel, fetchedData)
+		cd := gd.ChannelData
+		if cd == nil {
+			cd = make(map[string]ChannelData)
+		}
+		d := &ChannelData{}
+		err = json.Unmarshal([]byte(fetchedData), d)
+		if err != nil {
+			rgb.RPrintf("[%s] channel data could not be unmarshalled : %s\n", TimeStamp(), err.Error())
+		}
+		cd[channel] = *d
+		gd.ChannelData = cd
+		rgb.YPrintf("[%s] channel data : %v\n", TimeStamp(), gd.ChannelData)
+	}
+	// Managed to leave the main channel!?
+	rgb.YPrintf("[%s] Read channel data for %d channels\n", TimeStamp(), len(gd.ChannelData))
+	return nil
+}
+
+func (gd *GlobalData) ReadOrCreateChannelKey(channel string) string {
+	record := gd.ChannelData[channel]
+	magicCode := ""
+	if record.AdminKey == "" {
+		rgb.YPrintf("[%s] No channel key for #%s exists, creating one\n", TimeStamp(), channel)
+		newuu, _ := uuid.NewRandom()
+		magicCode = base64.StdEncoding.EncodeToString([]byte(newuu.String()))
+		gd.UpdateJoined(channel, true)
+		gd.UpdateChannelKey(channel, magicCode)
+		gd.UpdateChannelName(channel, channel)
+		rgb.YPrintf("[%s] Cached channel key for #%s\n", TimeStamp(), record.Name)
+	} else {
+		magicCode = record.AdminKey
+		rgb.YPrintf("[%s] Loaded data for #%s\n", TimeStamp(), channel)
+	}
+	return magicCode
+}
+
+const UTCFormat = "Jan 2 15:04:05 UTC"
+
+func TimeStamp() string {
+	return TimeStampFmt(UTCFormat)
+}
+
+func TimeStampFmt(format string) string {
+	return time.Now().Format(format)
+}
--- a/internal/irc/irc.go
+++ b/internal/irc/irc.go
@ -2,7 +2,6 @@ package irc

 import (
 	"bufio"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@ -15,9 +14,8 @@ import (
 	"strings"
 	"time"

+	data "git.martyn.berlin/martyn/twitchsingstools/internal/data"
 	rgb "github.com/foresthoffman/rgblog"
-	uuid "github.com/google/uuid"
-	scribble "github.com/nanobox-io/golang-scribble"
 )

 const UTCFormat = "Jan 2 15:04:05 UTC"
@ -58,15 +56,6 @@ type AppOAuthCred struct {
 	ClientSecret string `json:"client_secret,omitempty"`
 }

-type ConfigStruct struct {
-	InitialChannels []string `json:"channels"`
-	IrcOAuthPath    string   `json:"ircoauthpath,omitempty"`
-	StringPath      string   `json:"authpath,omitempty"`
-	DataPath        string   `json:"datapath,omitempty"`
-	ExternalUrl     string   `json:"externalurl,omitempty"`
-	AppOAuthPath    string   `json:"appoauthpath,omitempty"`
-}
-
 type KardBot struct {
 	Channel        string
 	conn           net.Conn
@ -80,57 +69,7 @@ type KardBot struct {
 	Server         string
 	startTime      time.Time
 	Prompts        []string
-	Database       scribble.Driver
-	ChannelData    map[string]ChannelData
-	Config         ConfigStruct
-}
-
-type SingsVideoStruct struct {
-	Date           time.Time `json:"date"`           // Golang date of creation
-	FullTitle      string    `json:"fullTitle"`      // Full Title
-	Duet           bool      `json:"duet"`           // Is it a duet?
-	OtherSinger    string    `json:"otherSinger"`    // Twitch NAME of the other singer, extracted from the title
-	SongTitle      string    `json:"songTitle"`      // extracted from title
-	LastSungSong   time.Time `json:"LastSungSong"`   // Last time this SONG was sung
-	LastSungSinger time.Time `json:"LastSungSinger"` // Last time a duet was sung with this SINGER, regardless of song, only Duets have this date initialised
-}
-
-// CommandType Kinda an enum
-type CommandType string
-
-const (
-	RandomSinger CommandType = "RandomSinger"
-	RandomPrompt CommandType = "RandomPrompt"
-	RandomSong   CommandType = "RandomSong"
-	AgingSinger  CommandType = "AgingSinger"
-	AgingSong    CommandType = "AgingSong"
-)
-
-func (ct CommandType) IsValid() error {
-	switch ct {
-	case RandomSinger, RandomPrompt, RandomSong, AgingSinger, AgingSong:
-		return nil
-	}
-	return errors.New("Invalid command type")
-}
-
-type CommandStruct struct {
-	CommandName CommandType `json:"commandName"`
-	KeyWord     string      `json:"keyword"`
-}
-
-type ChannelData struct {
-	ControlChannel    bool
-	Name              string             `json:"name"`
-	AdminKey          string             `json:"value,omitempty"`
-	Commands          []CommandStruct    `json:"commands,omitempty"`
-	ExtraStrings      string             `json:"extrastrings,omitempty"`
-	JoinTime          time.Time          `json:"jointime"`
-	HasLeft           bool               `json:"hasleft"`
-	VideoCache        []SingsVideoStruct `json:"videoCache"`
-	VideoCacheUpdated time.Time          `json:"videoCacheUpdated"`
-	Bearer            string             `json:"bearer"`
-	TwitchUserID      string             `json:"twitchUserID"`
+	GlobalData     data.GlobalData
 }

 // Connects the bot to the Twitch IRC server. The bot will continue to try to connect until it
@ -160,7 +99,7 @@ func (bb *KardBot) Disconnect() {
 // Look at the channels I'm actually in
 func (bb *KardBot) ActiveChannels() int {
 	count := 0
-	for _, channel := range bb.ChannelData {
+	for _, channel := range bb.GlobalData.ChannelData {
 		if !channel.HasLeft {
 			count = count + 1
 		}
@ -168,29 +107,6 @@ func (bb *KardBot) ActiveChannels() int {
 	return count
 }

-func (bb *KardBot) UpdateVideoCache(user string, videos []SingsVideoStruct) {
-	record := bb.ChannelData[user]
-	fmt.Printf("Replacing cache of %d performances with a new cache of %d performances", len(record.VideoCache), len(videos))
-	record.VideoCache = videos
-	record.VideoCacheUpdated = time.Now()
-	bb.Database.Write("channelData", user, record)
-	bb.ChannelData[user] = record
-}
-
-func (bb *KardBot) UpdateBearerToken(user string, token string) {
-	record := bb.ChannelData[user]
-	record.Bearer = token
-	bb.Database.Write("channelData", user, record)
-	bb.ChannelData[user] = record
-}
-
-func (bb *KardBot) UpdateTwitchUserID(user string, userid string) {
-	record := bb.ChannelData[user]
-	record.TwitchUserID = userid
-	bb.Database.Write("channelData", user, record)
-	bb.ChannelData[user] = record
-}
-
 // Listens for and logs messages from chat. Responds to commands from the channel owner. The bot
 // continues until it gets disconnected, told to shutdown, or forcefully shutdown.
 func (bb *KardBot) HandleChat() error {
@ -222,11 +138,7 @@ func (bb *KardBot) HandleChat() error {
 			matches := ConnectRegex.FindStringSubmatch(line)
 			if nil != matches {
 				realUserName := matches[1]
-				if bb.ChannelData[realUserName].Name == "" {
-					record := ChannelData{Name: realUserName, JoinTime: time.Now(), Commands: nil, ControlChannel: true}
-					bb.Database.Write("channelData", realUserName, record)
-					bb.ChannelData[realUserName] = record
-				}
+				bb.GlobalData.UpdateJoined(realUserName, false)
 				bb.JoinChannel(realUserName)
 			}

@ -258,13 +170,13 @@ func (bb *KardBot) HandleChat() error {
 					if nil != cmdMatches {
 						cmd := cmdMatches[1]
 						cardCommand := ""
-						commands := bb.ChannelData[channel].Commands
+						commands := bb.GlobalData.ChannelData[channel].Commands
 						for _, command := range commands {
-							if command.CommandName == RandomPrompt {
+							if command.CommandName == data.RandomPrompt {
 								cardCommand = command.KeyWord
 							}
 						}
-						rgb.YPrintf("[%s] Checking cmd %s against [%s]\n", TimeStamp(), cmd, bb.ChannelData[channel].Commands)
+						rgb.YPrintf("[%s] Checking cmd %s against [%s]\n", TimeStamp(), cmd, bb.GlobalData.ChannelData[channel].Commands)
 						switch cmd {
 						case cardCommand:
 							if cardCommand != "" {
@ -272,13 +184,9 @@ func (bb *KardBot) HandleChat() error {
 								bb.Say("Your prompt is : "+bb.Prompts[rand.Intn(len(bb.Prompts))], channel)
 							}
 						case "join":
-							if bb.ChannelData[channel].ControlChannel {
+							if bb.GlobalData.ChannelData[channel].ControlChannel {
 								rgb.CPrintf("[%s] Join asked for by %s on %s' channel!\n", TimeStamp(), userName, channel)
-								if bb.ChannelData[userName].Name == "" {
-									record := ChannelData{Name: userName, JoinTime: time.Now(), Commands: nil, ControlChannel: true}
-									bb.Database.Write("channelData", userName, record)
-									bb.ChannelData[userName] = record
-								}
+								bb.GlobalData.UpdateJoined(userName, false)
 								bb.JoinChannel(userName)
 							}
 						}
@ -295,7 +203,7 @@ func (bb *KardBot) HandleChat() error {
 								bb.Disconnect()
 								return nil
 							case "kcardadmin":
-								magicCode := bb.ReadOrCreateChannelKey(channel)
+								magicCode := bb.GlobalData.ReadOrCreateChannelKey(channel)
 								rgb.CPrintf(
 									"[%s] Magic code is %s - https://karaokards.ing.martyn.berlin/admin/%s/%s\n",
 									TimeStamp(),
@ -455,18 +363,11 @@ func (bb *KardBot) Start() {
 		return
 	}

-	err = bb.readChannelData()
-	if nil != err {
-		fmt.Println(err)
-		fmt.Println("Aborting!")
-		return
-	}
-
 	for {
 		bb.Connect()
 		bb.Login()
-		if len(bb.ChannelData) > 0 {
-			for channelName, channelData := range bb.ChannelData {
+		if len(bb.GlobalData.ChannelData) > 0 {
+			for channelName, channelData := range bb.GlobalData.ChannelData {
 				if !channelData.HasLeft {
 					bb.JoinChannel(channelName)
 				}
@ -487,74 +388,6 @@ func (bb *KardBot) Start() {
 	}
 }

-func (bb *KardBot) readChannelData() error {
-	records, err := bb.Database.ReadAll("channelData")
-	if err != nil {
-		// no db? initialise one?
-		record := ChannelData{Name: bb.Channel, JoinTime: time.Now(), Commands: nil}
-		rgb.YPrintf("[%s] No channel table for #%s exists, creating...\n", TimeStamp(), bb.Channel)
-		if err := bb.Database.Write("channelData", bb.Channel, record); err != nil {
-			return err
-		}
-		bb.ChannelData = make(map[string]ChannelData)
-		bb.ChannelData[bb.Channel] = record
-	} else {
-		bb.ChannelData = make(map[string]ChannelData)
-	}
-	for _, data := range records {
-		record := ChannelData{}
-		err := json.Unmarshal([]byte(data), &record)
-		if err != nil {
-			return err
-		}
-	}
-	// Managed to leave the main channel!?
-	if bb.ChannelData[bb.Channel].Name == "" {
-		rgb.YPrintf("[%s] No channel data for #%s exists, creating...\n", TimeStamp(), bb.Channel)
-		record := ChannelData{Name: bb.Channel, JoinTime: time.Now(), Commands: nil}
-		bb.ChannelData[bb.Channel] = record
-		if err := bb.Database.Write("channelData", bb.Channel, record); err != nil {
-			return err
-		}
-		records, err = bb.Database.ReadAll("channelData")
-	}
-	rgb.YPrintf("[%s] Read channel data for %d channels\n", TimeStamp(), len(bb.ChannelData))
-	return nil
-}
-
-func (bb *KardBot) ReadOrCreateChannelKey(channel string) string {
-	magicCode := ""
-	var err error
-	var record ChannelData
-	if record, ok := bb.ChannelData[channel]; !ok {
-		rgb.YPrintf("[%s] No channel data for #%s exists, creating\n", TimeStamp(), channel)
-		err = bb.Database.Read("channelData", channel, &record)
-		if err == nil {
-			bb.ChannelData[channel] = record
-		}
-	}
-	record = bb.ChannelData[channel]
-	if err != nil || record.AdminKey == "" {
-		rgb.YPrintf("[%s] No channel key for #%s exists, creating one\n", TimeStamp(), channel)
-		newuu, _ := uuid.NewRandom()
-		magicCode = base64.StdEncoding.EncodeToString([]byte(newuu.String()))
-		record.HasLeft = true
-		record.AdminKey = magicCode
-		if record.Name == "" {
-			record.Name = channel
-		}
-		if err := bb.Database.Write("channelData", channel, record); err != nil {
-			rgb.RPrintf("[%s] Error writing channel data for #%s\n", TimeStamp(), channel)
-		}
-		bb.ChannelData[record.Name] = record
-		rgb.YPrintf("[%s] Cached channel key for #%s\n", TimeStamp(), record.Name)
-	} else {
-		magicCode = record.AdminKey
-		rgb.YPrintf("[%s] Loaded data for #%s\n", TimeStamp(), channel)
-	}
-	return magicCode
-}
-
 func TimeStamp() string {
 	return TimeStampFmt(UTCFormat)
 }
--- a/internal/webserver/webserver.go
+++ b/internal/webserver/webserver.go
@ -9,6 +9,7 @@ import (
 	"sync"
 	"unicode"

+	data "git.martyn.berlin/martyn/twitchsingstools/internal/data"
 	irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc"
 	"github.com/dustin/go-humanize"
 	"github.com/gorilla/handlers"
@ -77,6 +78,7 @@ type videosResponse struct {
 }

 var ircBot *irc.KardBot
+var globalData *data.GlobalData

 func HealthHandler(response http.ResponseWriter, request *http.Request) {
 	response.Header().Add("Content-type", "text/plain")
@ -130,7 +132,7 @@ func TemplateHandler(response http.ResponseWriter, request *http.Request) {
 		// NotFoundHandler(response, request)
 		// return
 	}
-	var td = TemplateData{ircBot.Prompts[rand.Intn(len(ircBot.Prompts))], len(ircBot.Prompts), ircBot.ActiveChannels(), 0, ircBot.AppCredentials.ClientID, "https://" + ircBot.Config.ExternalUrl}
+	var td = TemplateData{ircBot.Prompts[rand.Intn(len(ircBot.Prompts))], len(ircBot.Prompts), ircBot.ActiveChannels(), 0, ircBot.AppCredentials.ClientID, "https://" + globalData.Config.ExternalURL}
 	err = tmpl.Execute(response, td)
 	if err != nil {
 		http.Error(response, err.Error(), http.StatusInternalServerError)
@ -153,6 +155,7 @@ func humanTimeFromTimeString(s string) string {
 type AugmentedSingsVideoStruct struct {
 	Date               time.Time
 	NiceDate           string
+	ShortDate          string
 	FullTitle          string
 	Duet               bool
 	OtherSinger        string
@ -161,12 +164,15 @@ type AugmentedSingsVideoStruct struct {
 	NiceLastSungSong   string
 	LastSungSinger     time.Time
 	NiceLastSungSinger string
+	VideoURL           string
+	VideoNumber        string //yes, I don't care any more.
 }

-func AugmentSingsVideoStructForCSV(input irc.SingsVideoStruct) AugmentedSingsVideoStruct {
+func AugmentSingsVideoStructForCSV(input data.SingsVideoStruct) AugmentedSingsVideoStruct {
 	var ret AugmentedSingsVideoStruct
 	ret.Date = input.Date
 	ret.NiceDate = input.Date.Format("2006-01-02 15:04:05")
+	ret.ShortDate = input.Date.Format("2006-01-02")
 	ret.FullTitle = input.FullTitle
 	ret.Duet = input.Duet
 	ret.OtherSinger = input.OtherSinger
@ -174,6 +180,9 @@ func AugmentSingsVideoStructForCSV(input irc.SingsVideoStruct) AugmentedSingsVid
 	ret.LastSungSong = input.LastSungSong
 	ret.NiceLastSungSong = input.LastSungSong.Format("2006-01-02 15:04:05")
 	ret.LastSungSinger = input.LastSungSinger
+	ret.VideoURL = input.VideoURL
+	urlParts := strings.Split(input.VideoURL, "/")
+	ret.VideoNumber = urlParts[len(urlParts)-1]
 	if !ret.Duet {
 		ret.NiceLastSungSinger = "Solo performance"
 	} else {
@ -182,7 +191,7 @@ func AugmentSingsVideoStructForCSV(input irc.SingsVideoStruct) AugmentedSingsVid
 	return ret
 }

-func AugmentSingsVideoStructSliceForCSV(input []irc.SingsVideoStruct) []AugmentedSingsVideoStruct {
+func AugmentSingsVideoStructSliceForCSV(input []data.SingsVideoStruct) []AugmentedSingsVideoStruct {
 	ret := make([]AugmentedSingsVideoStruct, 0)
 	for _, record := range input {
 		ret = append(ret, AugmentSingsVideoStructForCSV(record))
@ -190,7 +199,7 @@ func AugmentSingsVideoStructSliceForCSV(input []irc.SingsVideoStruct) []Augmente
 	return ret
 }

-func AugmentSingsVideoStruct(input irc.SingsVideoStruct) AugmentedSingsVideoStruct {
+func AugmentSingsVideoStruct(input data.SingsVideoStruct) AugmentedSingsVideoStruct {
 	var ret AugmentedSingsVideoStruct
 	ret.Date = input.Date
 	ret.NiceDate = humanize.Time(input.Date)
@ -209,7 +218,7 @@ func AugmentSingsVideoStruct(input irc.SingsVideoStruct) AugmentedSingsVideoStru
 	return ret
 }

-func AugmentSingsVideoStructSlice(input []irc.SingsVideoStruct) []AugmentedSingsVideoStruct {
+func AugmentSingsVideoStructSlice(input []data.SingsVideoStruct) []AugmentedSingsVideoStruct {
 	ret := make([]AugmentedSingsVideoStruct, 0)
 	for _, record := range input {
 		ret = append(ret, AugmentSingsVideoStruct(record))
@ -266,8 +275,8 @@ func ValidateTwitchBearerToken(bearer string) (bool, error) {
 	return resp.StatusCode == 200, nil
 }

-func twitchVidToSingsVid(twitchFormat videoStruct) (irc.SingsVideoStruct, error) {
-	var ret irc.SingsVideoStruct
+func twitchVidToSingsVid(twitchFormat videoStruct) (data.SingsVideoStruct, error) {
+	var ret data.SingsVideoStruct
 	layout := "2006-01-02T15:04:05Z"
 	var d time.Time
 	d, err := time.Parse(layout, twitchFormat.CreatedAt)
@ -275,6 +284,7 @@ func twitchVidToSingsVid(twitchFormat videoStruct) (irc.SingsVideoStruct, error)
 		return ret, err
 	}
 	ret.Date = d
+	ret.VideoURL = twitchFormat.URL

 	var DuetRegex = regexp.MustCompile(`^Duet with ([^ ]*): (.*)$`)
 	matches := DuetRegex.FindAllStringSubmatch(twitchFormat.Title, -1)
@ -307,7 +317,7 @@ type SingerSings struct {
 	Sings      int
 }

-func calculateTopNSongs(songCache []irc.SingsVideoStruct, howMany int) []SongSings {
+func calculateTopNSongs(songCache []data.SingsVideoStruct, howMany int) []SongSings {
 	songMap := map[string]int{}
 	for _, record := range songCache {
 		sings := songMap[record.SongTitle]
@ -338,7 +348,7 @@ func IsLower(s string) bool {
 	return true
 }

-func unmangleSingerName(MangledCaseName string, songCache []irc.SingsVideoStruct) string {
+func unmangleSingerName(MangledCaseName string, songCache []data.SingsVideoStruct) string {
 	options := make(map[string]string, 0)
 	for _, record := range songCache {
 		if strings.ToUpper(MangledCaseName) == strings.ToUpper(record.OtherSinger) {
@ -413,7 +423,7 @@ type CacheDetails struct {

 func getCacheDetails(channel string) CacheDetails {
 	var ret CacheDetails
-	channelData := ircBot.ChannelData[channel]
+	channelData := globalData.ChannelData[channel]
 	ret.Age = time.Now().Sub(channelData.VideoCacheUpdated)
 	ret.AgeStr = humanize.Time(channelData.VideoCacheUpdated)
 	ret.SongCount = len(channelData.VideoCache)
@ -422,36 +432,36 @@ func getCacheDetails(channel string) CacheDetails {

 func forceUpdateCache(channel string) {
 	fmt.Printf("Forcing cache update!")
-	channelData := ircBot.ChannelData[channel]
+	channelData := globalData.ChannelData[channel]
 	tenHours := time.Hour * -10
 	videoCacheUpdated := time.Now().Add(tenHours) // Subtract 10 hours from now, cache is 10 hours old.
 	channelData.VideoCacheUpdated = videoCacheUpdated
-	ircBot.ChannelData[channel] = channelData
+	globalData.ChannelData[channel] = channelData
 	updateCacheIfNecessary(channel)
 }

 func updateCacheIfNecessary(channel string) {
 	cacheLock.Lock()
-	channelData := ircBot.ChannelData[channel]
+	channelData := globalData.ChannelData[channel]
 	if time.Now().Sub(channelData.VideoCacheUpdated).Hours() > 1 {
-		fmt.Printf("Cache of %d performances is older than an hour - %.1f hours old to be precise... fetching.\n", len(channelData.VideoCache), time.Now().Sub(ircBot.ChannelData[channel].VideoCacheUpdated).Hours())
+		fmt.Printf("Cache of %d performances is older than an hour - %.1f hours old to be precise... fetching.\n", len(channelData.VideoCache), time.Now().Sub(globalData.ChannelData[channel].VideoCacheUpdated).Hours())
 		vids, err := fetchAllVoDs(channelData.TwitchUserID, channelData.Bearer)
 		if err != nil {
-			errCache := make([]irc.SingsVideoStruct, 0)
-			var ret irc.SingsVideoStruct
+			errCache := make([]data.SingsVideoStruct, 0)
+			var ret data.SingsVideoStruct
 			ret.FullTitle = "Error fetching videos: " + err.Error()
 			errCache = append(errCache, ret)
 			vids = errCache
 		}
 		updateCalculatedFields(vids)
-		ircBot.UpdateVideoCache(channel, vids)
+		globalData.UpdateVideoCache(channel, vids)
 	} else {
-		fmt.Printf("Cache of %d performances is younger than an hour - %.1f hours old to be precise... not fetching.\n", len(channelData.VideoCache), time.Now().Sub(ircBot.ChannelData[channel].VideoCacheUpdated).Hours())
+		fmt.Printf("Cache of %d performances is younger than an hour - %.1f hours old to be precise... not fetching.\n", len(channelData.VideoCache), time.Now().Sub(globalData.ChannelData[channel].VideoCacheUpdated).Hours())
 	}
 	cacheLock.Unlock()
 }

-func calculateTopNSingers(songCache []irc.SingsVideoStruct, howMany int) []SingerSings {
+func calculateTopNSingers(songCache []data.SingsVideoStruct, howMany int) []SingerSings {
 	songMap := map[string]int{}
 	songCount := 0
 	for _, record := range songCache {
@ -475,7 +485,7 @@ func calculateTopNSingers(songCache []irc.SingsVideoStruct, howMany int) []Singe
 	return slice
 }

-func calculateLastSungSongDate(songCache []irc.SingsVideoStruct, SongTitle string) time.Time {
+func calculateLastSungSongDate(songCache []data.SingsVideoStruct, SongTitle string) time.Time {
 	var t time.Time
 	for _, record := range songCache {
 		if record.SongTitle == SongTitle {
@ -487,7 +497,7 @@ func calculateLastSungSongDate(songCache []irc.SingsVideoStruct, SongTitle strin
 	return t
 }

-func calculateLastSungSingerDate(songCache []irc.SingsVideoStruct, Singer string) time.Time {
+func calculateLastSungSingerDate(songCache []data.SingsVideoStruct, Singer string) time.Time {
 	var t time.Time
 	for _, record := range songCache {
 		if strings.ToUpper(record.OtherSinger) == strings.ToUpper(Singer) {
@ -502,7 +512,7 @@ func calculateLastSungSingerDate(songCache []irc.SingsVideoStruct, Singer string
 	return t
 }

-func updateCalculatedFields(songCache []irc.SingsVideoStruct) {
+func updateCalculatedFields(songCache []data.SingsVideoStruct) {
 	for i, record := range songCache {
 		if record.Duet {
 			songCache[i].LastSungSinger = calculateLastSungSingerDate(songCache, record.OtherSinger)
@ -511,7 +521,7 @@ func updateCalculatedFields(songCache []irc.SingsVideoStruct) {
 	}
 }

-func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.SingsVideoStruct, error) {
+func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]data.SingsVideoStruct, error) {
 	url := ""
 	if from == "" {
 		url = "videos?user_id=" + userID + "&first=100&type=upload"
@ -529,7 +539,7 @@ func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.S
 	if err != nil {
 		return nil, err
 	}
-	titles := make([]irc.SingsVideoStruct, 0)
+	titles := make([]data.SingsVideoStruct, 0)
 	for _, videoData := range fullResponse.Data {
 		ret, err := twitchVidToSingsVid(videoData)
 		if err != nil {
@ -549,7 +559,7 @@ func fetchVoDsPagesRecursive(userID string, bearer string, from string) ([]irc.S
 	return titles, nil
 }

-func fetchAllVoDs(userID string, bearer string) ([]irc.SingsVideoStruct, error) {
+func fetchAllVoDs(userID string, bearer string) ([]data.SingsVideoStruct, error) {
 	tokenValid, err := ValidateTwitchBearerToken(bearer)
 	if err != nil {
 		fmt.Println("Error validating token : " + err.Error())
@ -561,7 +571,7 @@ func fetchAllVoDs(userID string, bearer string) ([]irc.SingsVideoStruct, error)
 	}
 	titles, err := fetchVoDsPagesRecursive(userID, bearer, "")
 	if err != nil {
-		return make([]irc.SingsVideoStruct, 0), err
+		return make([]data.SingsVideoStruct, 0), err
 	}
 	return titles, nil
 }
@ -578,7 +588,7 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
 				"client_secret": {ircBot.AppCredentials.ClientSecret},
 				"code":          {vars["code"]},
 				"grant_type":    {"authorization_code"},
-				"redirect_uri":  {"https://" + ircBot.Config.ExternalUrl + "/twitchadmin"}})
+				"redirect_uri":  {"https://" + globalData.Config.ExternalURL + "/twitchadmin"}})
 		if err != nil {
 			response.WriteHeader(500)
 			response.Header().Add("Content-type", "text/plain")
@ -629,10 +639,10 @@ func TwitchAdminHandler(response http.ResponseWriter, request *http.Request) {
 		}

 		user := usersObject.Data[0]
-		magicCode := ircBot.ReadOrCreateChannelKey(user.Login)
-		ircBot.UpdateBearerToken(user.Login, oauthResponse.Access_token)
-		ircBot.UpdateTwitchUserID(user.Login, user.Id)
-		url := "https://" + ircBot.Config.ExternalUrl + "/admin/" + user.Login + "/" + magicCode
+		magicCode := globalData.ReadOrCreateChannelKey(user.Login)
+		globalData.UpdateBearerToken(user.Login, oauthResponse.Access_token)
+		globalData.UpdateTwitchUserID(user.Login, user.Id)
+		url := "https://" + globalData.Config.ExternalURL + "/admin/" + user.Login + "/" + magicCode
 		http.Redirect(response, request, url, http.StatusFound)

 	} else {
@ -664,7 +674,7 @@ func TwitchBackendHandler(response http.ResponseWriter, request *http.Request) {
 		// ircBot.AppCredentials.Password
 		// vars["oauthtoken"]
 		// authorization_code
-		// "https://"+ircBot.Config.ExternalUrl+/twitchadmin
+		// "https://"+globalData.Config.ExternalURL+/twitchadmin
 		fmt.Println("Asking twitch for more...")
 		resp, err := http.PostForm(
 			"https://id.twitch.tv/oauth2/token",
@ -673,7 +683,7 @@ func TwitchBackendHandler(response http.ResponseWriter, request *http.Request) {
 				"client_secret": {ircBot.AppCredentials.ClientSecret},
 				"code":          {vars["code"]},
 				"grant_type":    {"authorization_code"},
-				"redirect_uri":  {"https://" + ircBot.Config.ExternalUrl + "/twitchadmin"}})
+				"redirect_uri":  {"https://" + globalData.Config.ExternalURL + "/twitchadmin"}})
 		if err != nil {
 			response.Header().Add("Content-type", "text/plain")
 			fmt.Fprint(response, "ERROR: "+err.Error())
@ -693,13 +703,13 @@ func TwitchBackendHandler(response http.ResponseWriter, request *http.Request) {

 func CSVHandler(response http.ResponseWriter, request *http.Request) {
 	vars := mux.Vars(request)
-	if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
+	if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey {
 		UnauthorizedHandler(response, request)
 		return
 	}
 	type TemplateData struct {
 		Channel      string
-		Commands     []irc.CommandStruct
+		Commands     []data.CommandStruct
 		ExtraStrings string
 		SinceTime    time.Time
 		SinceTimeUTC string
@ -710,7 +720,7 @@ func CSVHandler(response http.ResponseWriter, request *http.Request) {
 		TopNSingers  []SingerSings
 	}
 	updateCacheIfNecessary(vars["channel"])
-	channelData := ircBot.ChannelData[vars["channel"]]
+	channelData := globalData.ChannelData[vars["channel"]]
 	topNSongs := calculateTopNSongs(channelData.VideoCache, 10)
 	topNSingers := calculateTopNSingers(channelData.VideoCache, 10)
 	var td = TemplateData{channelData.Name, channelData.Commands, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft, AugmentSingsVideoStructSliceForCSV(channelData.VideoCache), topNSongs, topNSingers}
@ -729,13 +739,14 @@ func CSVHandler(response http.ResponseWriter, request *http.Request) {

 func JSONHandler(response http.ResponseWriter, request *http.Request) {
 	vars := mux.Vars(request)
-	if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
+	if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey {
+		fmt.Printf("%s != %s\n", vars["key"], globalData.ChannelData[vars["channel"]].AdminKey)
 		UnauthorizedHandler(response, request)
 		return
 	}
 	type TemplateData struct {
 		Channel      string
-		Commands     []irc.CommandStruct
+		Commands     []data.CommandStruct
 		ExtraStrings string
 		SinceTime    time.Time
 		SinceTimeUTC string
@ -746,7 +757,7 @@ func JSONHandler(response http.ResponseWriter, request *http.Request) {
 		TopNSingers  []SingerSings
 	}
 	updateCacheIfNecessary(vars["channel"])
-	channelData := ircBot.ChannelData[vars["channel"]]
+	channelData := globalData.ChannelData[vars["channel"]]

 	var topNSongs []SongSings
 	var topNSingers []SingerSings
@ -769,9 +780,45 @@ func JSONHandler(response http.ResponseWriter, request *http.Request) {
 	}
 }

+func ScriptHandler(response http.ResponseWriter, request *http.Request) {
+	vars := mux.Vars(request)
+	if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey {
+		UnauthorizedHandler(response, request)
+		return
+	}
+	type TemplateData struct {
+		Channel      string
+		Commands     []data.CommandStruct
+		ExtraStrings string
+		SinceTime    time.Time
+		SinceTimeUTC string
+		Leaving      bool
+		HasLeft      bool
+		SongData     []AugmentedSingsVideoStruct
+		TopNSongs    []SongSings
+		TopNSingers  []SingerSings
+	}
+	updateCacheIfNecessary(vars["channel"])
+	channelData := globalData.ChannelData[vars["channel"]]
+	topNSongs := calculateTopNSongs(channelData.VideoCache, 10)
+	topNSingers := calculateTopNSingers(channelData.VideoCache, 10)
+	var td = TemplateData{channelData.Name, channelData.Commands, channelData.ExtraStrings, channelData.JoinTime, channelData.JoinTime.Format(irc.UTCFormat), false, channelData.HasLeft, AugmentSingsVideoStructSliceForCSV(channelData.VideoCache), topNSongs, topNSingers}
+	if request.URL.Path[0:11] == "/script.bat" {
+		response.Header().Add("Content-Disposition", "attachment; filename=\"script.bat\"")
+		response.Header().Add("Content-type", "application/x-bat")
+		tmpl := template.Must(template.ParseFiles("web/script.bat"))
+		tmpl.Execute(response, td)
+	} else {
+		response.Header().Add("Content-Disposition", "attachment; filename=\"script.sh\"")
+		response.Header().Add("Content-type", "text/x-shellscript")
+		tmpl := template.Must(template.ParseFiles("web/script.sh"))
+		tmpl.Execute(response, td)
+	}
+}
+
 func CacheDetailsHandler(response http.ResponseWriter, request *http.Request) {
 	vars := mux.Vars(request)
-	if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
+	if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey {
 		UnauthorizedHandler(response, request)
 		return
 	}
@ -783,23 +830,23 @@ func CacheDetailsHandler(response http.ResponseWriter, request *http.Request) {

 func BotDetailsHandler(response http.ResponseWriter, request *http.Request) {
 	vars := mux.Vars(request)
-	if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
+	if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey {
 		UnauthorizedHandler(response, request)
 		return
 	}
 	type ChannelDataSmaller struct {
-		Commands          []irc.CommandStruct `json:"commands,omitempty"`
-		ExtraStrings      string              `json:"extrastrings,omitempty"`
-		JoinTime          time.Time           `json:"jointime"`
-		HasLeft           bool                `json:"hasleft"`
-		VideoCacheUpdated time.Time           `json:"videoCacheUpdated"`
+		Commands          []data.CommandStruct `json:"commands,omitempty"`
+		ExtraStrings      string               `json:"extrastrings,omitempty"`
+		JoinTime          time.Time            `json:"jointime"`
+		HasLeft           bool                 `json:"hasleft"`
+		VideoCacheUpdated time.Time            `json:"videoCacheUpdated"`
 	}
 	var deets ChannelDataSmaller
-	deets.Commands = ircBot.ChannelData[vars["channel"]].Commands
-	deets.ExtraStrings = ircBot.ChannelData[vars["channel"]].ExtraStrings
-	deets.JoinTime = ircBot.ChannelData[vars["channel"]].JoinTime
-	deets.HasLeft = ircBot.ChannelData[vars["channel"]].HasLeft
-	deets.VideoCacheUpdated = ircBot.ChannelData[vars["channel"]].VideoCacheUpdated
+	deets.Commands = globalData.ChannelData[vars["channel"]].Commands
+	deets.ExtraStrings = globalData.ChannelData[vars["channel"]].ExtraStrings
+	deets.JoinTime = globalData.ChannelData[vars["channel"]].JoinTime
+	deets.HasLeft = globalData.ChannelData[vars["channel"]].HasLeft
+	deets.VideoCacheUpdated = globalData.ChannelData[vars["channel"]].VideoCacheUpdated
 	response.Header().Add("Content-type", "application/json")
 	enc := json.NewEncoder(response)
 	enc.Encode(deets)
@ -815,7 +862,7 @@ func ReactIndexHandler(entrypoint string) func(w http.ResponseWriter, r *http.Re

 func ForceRefreshHandler(response http.ResponseWriter, request *http.Request) {
 	vars := mux.Vars(request)
-	if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
+	if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey {
 		UnauthorizedHandler(response, request)
 		return
 	}
@ -827,21 +874,17 @@ func ForceRefreshHandler(response http.ResponseWriter, request *http.Request) {

 func JoinHandler(response http.ResponseWriter, request *http.Request) {
 	vars := mux.Vars(request)
-	if vars["key"] != ircBot.ChannelData[vars["channel"]].AdminKey {
+	if vars["key"] != globalData.ChannelData[vars["channel"]].AdminKey {
 		UnauthorizedHandler(response, request)
 		return
 	}
-	record := ircBot.ChannelData[vars["channel"]]
-	record.Name = vars["channel"]
-	record.JoinTime = time.Now()
-	record.HasLeft = false
-	ircBot.Database.Write("channelData", vars["channel"], record)
-	ircBot.ChannelData[vars["channel"]] = record
-	ircBot.JoinChannel(record.Name)
+	globalData.UpdateJoined(vars["channel"], false)
+	ircBot.JoinChannel(vars["channel"])
 }

-func HandleHTTP(passedIrcBot *irc.KardBot) {
+func HandleHTTP(passedIrcBot *irc.KardBot, passedGlobalData *data.GlobalData) {
 	ircBot = passedIrcBot
+	globalData = passedGlobalData
 	r := mux.NewRouter()
 	loggedRouter := handlers.LoggingHandler(os.Stdout, r)
 	r.NotFoundHandler = http.HandlerFunc(NotFoundHandler)
@ -857,6 +900,7 @@ func HandleHTTP(passedIrcBot *irc.KardBot) {
 	r.HandleFunc("/debug/{channel}/{key}", JSONHandler)
 	r.HandleFunc("/topsongs/{channel}/{key}", JSONHandler)
 	r.HandleFunc("/topsingers/{channel}/{key}", JSONHandler)
+	r.HandleFunc("/script.bat/{channel}/{key}", ScriptHandler)
 	r.Path("/twitchtobackend").Queries("access_token", "{access_token}", "scope", "{scope}", "token_type", "{token_type}").HandlerFunc(TwitchBackendHandler)
 	r.Path("/twitchadmin").Queries("code", "{code}", "scope", "{scope}").HandlerFunc(TwitchAdminHandler)
 	r.PathPrefix("/static").Handler(http.StripPrefix("/static", http.FileServer(http.Dir("./web/react-frontend/static"))))
--- a/internal/webserver/webserver_test.go
+++ b/internal/webserver/webserver_test.go
@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"

-	irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc"
+	data "git.martyn.berlin/martyn/twitchsingstools/internal/data"
 )

 func TestDateRegex(t *testing.T) {
@ -72,9 +72,9 @@ func TestSoloRegexes(t *testing.T) {
 }

 func TestCalculatedDates(t *testing.T) {
-	var record irc.SingsVideoStruct
+	var record data.SingsVideoStruct
 	format := "2006-01-02 15:04:05 +0000 UTC"
-	var mockCache []irc.SingsVideoStruct
+	var mockCache []data.SingsVideoStruct
 	record.Date, _ = time.Parse(format, "2020-07-13 19:43:11 +0000 UTC")
 	record.SongTitle = "Words Fail"
 	record.OtherSinger = "FullOfEmily"
--- a/main.go
+++ b/main.go
@ -2,6 +2,7 @@ package main

 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"os"
@ -9,6 +10,7 @@ import (
 	"time"

 	builtins "git.martyn.berlin/martyn/twitchsingstools/internal/builtins"
+	data "git.martyn.berlin/martyn/twitchsingstools/internal/data"
 	irc "git.martyn.berlin/martyn/twitchsingstools/internal/irc"
 	webserver "git.martyn.berlin/martyn/twitchsingstools/internal/webserver"
 	rgb "github.com/foresthoffman/rgblog"
@ -23,7 +25,7 @@ var selectablePrompts []string

 var customStrings customStringsStruct

-var config irc.ConfigStruct
+var config data.ConfigStruct

 func readConfig() {
 	var data []byte
@ -174,12 +176,32 @@ func main() {
 		}
 	} else {
 		if _, err := os.Stat(config.AppOAuthPath); os.IsNotExist(err) {
-			rgb.YPrintf("[%s] Error config-specified oauth file %s doesn't exist, bailing!\n", irc.TimeStamp(), config.AppOAuthPath)
+			rgb.RPrintf("[%s] Error config-specified oauth file %s doesn't exist, bailing!\n", irc.TimeStamp(), config.AppOAuthPath)
 			os.Exit(1)
 		}
 		appOauthPath = config.AppOAuthPath
 	}

+	rgb.YPrintf("[%s] Starting connection to redis...\n", irc.TimeStamp())
+	//TODO: unhardcode this
+	if os.Getenv("TSTOOLS_REDIS_HOST") != "" {
+		config.DatabaseSVC = os.Getenv("TSTOOLS_REDIS_HOST")
+	} else {
+		// assume localhost, which should fail.
+		config.DatabaseSVC = "localhost"
+	}
+	var globalData data.GlobalData
+	globalData.Config = config
+	globalData.ConnectDatabase()
+	defer globalData.Database.Close()
+	rgb.GPrintf("[%s] Connected to \"redis\" %s\n", irc.TimeStamp(), "config.DatabaseSVC")
+	err := globalData.ReadChannelData()
+	if nil != err {
+		fmt.Println(err)
+		fmt.Println("Aborting!")
+		os.Exit(1)
+	}
+	rgb.GPrintf("[%s] Read the channel data from \"redis\" successfully, now have %d records\n", irc.TimeStamp(), len(globalData.ChannelData))
 	// Replace the channel name, bot name, and the path to the private directory with your respective
 	// values.
 	var myBot irc.KardBot
@ -193,13 +215,12 @@ func main() {
 			AppPrivatePath: appOauthPath,
 			Server:         "irc.chat.twitch.tv",
 			Prompts:        selectablePrompts,
-			Database:       *persistentData,
-			Config:         config,
+			GlobalData:     globalData,
 		}
 	}
 	go func() {
 		rgb.YPrintf("[%s] Starting webserver on port %s\n", irc.TimeStamp(), "5353")
-		webserver.HandleHTTP(&myBot)
+		webserver.HandleHTTP(&myBot, &globalData)
 	}()
 	if ircOauthPath != "" {
 		myBot.Start()
--- a/web/script.bat
+++ b/web/script.bat
@ -0,0 +1,5 @@
+@ECHO OFF
+{{ range .SongData -}}
+IF NOT EXIST "{{.OtherSinger}}\" MKDIR {{.OtherSinger}}
+IF NOT EXIST "{{.OtherSinger}}\{{.ShortDate}}-{{.OtherSinger}}-{{.SongTitle}}-{{.VideoNumber}}.mp4" youtube-dl -o "{{.OtherSinger}}\{{.ShortDate}}-{{.OtherSinger}}-{{.SongTitle}}-{{.VideoNumber}}.mp4" {{.VideoURL}}
+{{ end }}
Author	SHA1	Message	Date
Martyn	1f00cd699f	Add link to BockTown script continuous-integration/drone/tag Build is passing Details continuous-integration/drone/push Build is passing Details Signed-off-by: Martyn Ranyard <m@rtyn.berlin>	2020-09-08 21:25:59 +02:00
Martyn	f7b0f074cf	Redis refactor and video download script. continuous-integration/drone/tag Build is passing Details Signed-off-by: Martyn Ranyard <m@rtyn.berlin>	2020-09-05 18:14:56 +02:00
Martyn	50303b3918	Production to external hosting continuous-integration/drone/push Build is passing Details Signed-off-by: Martyn Ranyard <m@rtyn.berlin>	2020-08-10 12:10:58 +02:00