Finished template + refac state.go + refac periodic tasks #8

- Email.AddPart now checks whether an encoded attachment exceeds the max number of bytes for an attachment. If this is the case and Part.DropIfBig is not set then an error will be returned, otherwise the Part will not be added to the Email silently (02/02/2023 - 10:48:39) - Added the Finished TemplatePath and the FinishedContext (02/02/2023 - 10:49:04) - Moved all the PathsToBytes stuff to a new file in main along with some further documentation on the new PathsToBytes interface introduced previously (02/02/2023 - 10:59:08) - Moved all cached field types and symbols to the new cached_fields.go file in main (02/02/2023 - 11:32:59) - Implemented sends for all the new templates into the Scout procedure (02/02/2023 - 12:31:31) - Fully implemented the Finished template (02/02/2023 - 13:43:17) - Periodic tasks are now registered from the config using a new field within TaskConfig called PeriodicTaskSignatures (02/02/2023 - 14:24:02) - Updates to tasks.StartServer to display registered tasks + updates to errors (02/02/2023 - 14:51:04)
andygello555 · Feb 2, 2023 · 6b91d1f · 6b91d1f
1 parent b8ca9e1
commit 6b91d1f
Show file tree

Hide file tree

Showing 14 changed files with 1,563 additions and 1,275 deletions.
diff --git a/cached_fields.go b/cached_fields.go
diff --git a/config.go b/config.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/RichardKnop/machinery/v1/tasks"
 	"github.com/andygello555/game-scout/db"
 	"github.com/andygello555/game-scout/db/models"
 	"github.com/andygello555/game-scout/email"
@@ -215,24 +216,49 @@ func (c *RedisConfig) String() string {
 	)
 }
 
+type PeriodicTaskSignature struct {
+	Args       []tasks.Arg `json:"args"`
+	Cron       string      `json:"cron"`
+	RetryCount int         `json:"retry_count"`
+}
+
+func (pts PeriodicTaskSignature) PeriodicTaskSignatureArgs() []tasks.Arg { return pts.Args }
+func (pts PeriodicTaskSignature) PeriodicTaskSignatureCron() string      { return pts.Cron }
+func (pts PeriodicTaskSignature) PeriodicTaskSignatureRetryCount() int   { return pts.RetryCount }
+
+func (pts PeriodicTaskSignature) String() string {
+	return fmt.Sprintf("{Args: %v, Cron: %v, RetryCount: %v}", pts.Args, pts.Cron, pts.RetryCount)
+}
+
 type TaskConfig struct {
-	DefaultQueue    string       `json:"default_queue"`
-	ResultsExpireIn int          `json:"results_expire_in"`
-	Broker          string       `json:"broker"`
-	ResultBackend   string       `json:"result_backend"`
-	Redis           *RedisConfig `json:"redis"`
+	DefaultQueue           string                           `json:"default_queue"`
+	ResultsExpireIn        int                              `json:"results_expire_in"`
+	Broker                 string                           `json:"broker"`
+	ResultBackend          string                           `json:"result_backend"`
+	Redis                  *RedisConfig                     `json:"redis"`
+	PeriodicTaskSignatures map[string]PeriodicTaskSignature `json:"periodic_task_signatures"`
 }
 
 func (c *TaskConfig) TasksDefaultQueue() string    { return c.DefaultQueue }
 func (c *TaskConfig) TasksResultsExpireIn() int    { return c.ResultsExpireIn }
 func (c *TaskConfig) TasksBroker() string          { return c.Broker }
 func (c *TaskConfig) TasksResultBackend() string   { return c.ResultBackend }
 func (c *TaskConfig) TasksRedis() task.RedisConfig { return c.Redis }
+func (c *TaskConfig) TasksPeriodicTaskSignatures() map[string]task.PeriodicTaskSignature {
+	periodicTaskSignatures := make(map[string]task.PeriodicTaskSignature)
+	for name, sig := range c.PeriodicTaskSignatures {
+		periodicTaskSignatures[name] = sig
+	}
+	return periodicTaskSignatures
+}
+func (c *TaskConfig) TasksPeriodicTaskSignature(taskName string) task.PeriodicTaskSignature {
+	return c.PeriodicTaskSignatures[taskName]
+}
 
 func (c *TaskConfig) String() string {
 	return fmt.Sprintf(
-		"{DefaultQueue: %v, ResultsExpireIn: %v, Broker: %v, ResultBackend: %v, Redis: %v}",
-		c.DefaultQueue, c.ResultsExpireIn, c.Broker, c.ResultBackend, c.Redis,
+		"{DefaultQueue: %v, ResultsExpireIn: %v, Broker: %v, ResultBackend: %v, Redis: %v, PeriodicTaskSignatures: %v}",
+		c.DefaultQueue, c.ResultsExpireIn, c.Broker, c.ResultBackend, c.Redis, c.PeriodicTaskSignatures,
 	)
 }
 
@@ -320,6 +346,8 @@ type ScrapeConfig struct {
 	Debug bool `json:"debug"`
 	// Storefronts is a list of StorefrontConfig that contains the configs for each models.Storefront.
 	Storefronts []*StorefrontConfig `json:"storefronts"`
+	// Constants are all the constants used throughout the Scout procedure as well as the ScoutWebPipes co-process.
+	Constants map[string]float64
 }
 
 func (sc *ScrapeConfig) ScrapeDebug() bool { return sc.Debug }

diff --git a/email/contexts.go b/email/contexts.go
@@ -25,7 +25,7 @@ type Context interface {
 	// Execute executes the Context via Template.Execute.
 	Execute() *Template
 	// AdditionalParts returns any additional Part to add onto an Email.
-	AdditionalParts() []Part
+	AdditionalParts() ([]Part, error)
 }
 
 // MeasureContext is a Context that contains the data required to fill out the Measure HTML template.
@@ -39,10 +39,10 @@ type MeasureContext struct {
 	Config                 Config
 }
 
-func (m *MeasureContext) Path() TemplatePath      { return Measure }
-func (m *MeasureContext) Execute() *Template      { return m.Template().Execute() }
-func (m *MeasureContext) AdditionalParts() []Part { return []Part{} }
-func (m *MeasureContext) Template() *Template     { return NewParsedTemplate(HTML, m).Template(m) }
+func (m *MeasureContext) Path() TemplatePath               { return Measure }
+func (m *MeasureContext) Execute() *Template               { return m.Template().Execute() }
+func (m *MeasureContext) AdditionalParts() ([]Part, error) { return []Part{}, nil }
+func (m *MeasureContext) Template() *Template              { return NewParsedTemplate(HTML, m).Template(m) }
 func (m *MeasureContext) Funcs() template.FuncMap {
 	return map[string]any{
 		"intRange": func(start, end, step int) []int {
@@ -130,14 +130,25 @@ func (m *MeasureContext) Funcs() template.FuncMap {
 	}
 }
 
-type ErrorContext struct {
+type FinishedContext struct {
+	BatchSize       int
+	DiscoveryTweets int
+	Started         time.Time
+	Finished        time.Time
+	Result          *models.ScoutResult
 }
 
-func (e *ErrorContext) Path() TemplatePath      { return Error }
-func (e *ErrorContext) Execute() *Template      { return e.Template().Execute() }
-func (e *ErrorContext) AdditionalParts() []Part { return []Part{} }
-func (e *ErrorContext) Template() *Template     { return NewParsedTemplate(Text, e).Template(e) }
-func (e *ErrorContext) Funcs() template.FuncMap {
-	//TODO implement me
-	panic("implement me")
+func (f *FinishedContext) Path() TemplatePath               { return Finished }
+func (f *FinishedContext) Execute() *Template               { return f.Template().Execute() }
+func (f *FinishedContext) Template() *Template              { return NewParsedTemplate(Text, f).Template(f) }
+func (f *FinishedContext) AdditionalParts() ([]Part, error) { return []Part{}, nil }
+func (f *FinishedContext) Funcs() template.FuncMap {
+	return map[string]any{
+		"stamp": func(t time.Time) string {
+			return t.Format(time.Stamp)
+		},
+		"duration": func(start time.Time, end time.Time) time.Duration {
+			return start.Sub(end)
+		},
+	}
 }
diff --git a/email/email.go b/email/email.go
@@ -25,6 +25,7 @@ import (
 const (
 	defaultBufSize                 = 4 * 1024
 	compressionLimit               = 15 * 1000 * 1024
+	maxAttachmentSize              = 24 * 1000 * 1024
 	maxContentTypeDetectionBufSize = 512
 )
 
@@ -74,6 +75,8 @@ type Part struct {
 	Attachment bool
 	// Filename is the filename of the attachment.
 	Filename string
+	// DropIfBig will drop the attachment from the email if its base64 encoded version larger than 25MB.
+	DropIfBig bool
 }
 
 // ContentTypeSlug returns the slugified version of the ContentType that can be used within the names of the temporary
@@ -344,6 +347,14 @@ func (e *Email) AddPart(part Part) (err error) {
 	}
 
 	reader := bufio.NewReader(part.Buffer)
+	if size := reader.Size(); size > maxAttachmentSize && part.Attachment {
+		if part.DropIfBig {
+			return
+		}
+		err = fmt.Errorf("attachment %q is bigger than %d bytes, and DropIfBig is not set", part.Filename, size)
+		return
+	}
+
 	nBytes, nChunks := int64(0), int64(0)
 	buf := make([]byte, 0, defaultBufSize)
 	for {

diff --git a/email/template.go b/email/template.go
@@ -35,9 +35,10 @@ const templateDir = "templates/"
 type TemplatePath string
 
 const (
-	Measure TemplatePath = templateDir + "measure.html"
-	Started TemplatePath = templateDir + "started.txt"
-	Error   TemplatePath = templateDir + "error.txt"
+	Measure  TemplatePath = templateDir + "measure.html"
+	Started  TemplatePath = templateDir + "started.txt"
+	Error    TemplatePath = templateDir + "error.txt"
+	Finished TemplatePath = templateDir + "finished.txt"
 )
 
 // TemplatePathFromName returns the TemplatePath of the given name.
@@ -49,6 +50,8 @@ func TemplatePathFromName(name string) TemplatePath {
 		return Started
 	case "error":
 		return Error
+	case "finished":
+		return Finished
 	default:
 		panic(fmt.Errorf("there is no template with name %q", name))
 	}
@@ -63,6 +66,8 @@ func (tt TemplatePath) Name() string {
 		return "Started"
 	case Error:
 		return "Error"
+	case Finished:
+		return "Finished"
 	default:
 		return "Unknown"
 	}
@@ -502,8 +507,11 @@ func (t *Template) Email() (email *Email, err error) {
 	}
 
 	// Add any additional parts
-	if additionalParts := t.Context.AdditionalParts(); len(additionalParts) > 0 {
+	var additionalParts []Part
+	if additionalParts, err = t.Context.AdditionalParts(); len(additionalParts) > 0 && err == nil {
 		errs = append(errs, email.AddParts(additionalParts...))
+	} else if err != nil {
+		errs = append(errs, errors.Wrapf(err, "could not find additional parts for %sContext", t.Path.Name()))
 	}
 
 	// Merge all the errors. If this returns a non-nil error then we will Close the email

diff --git a/email/templates/error.txt b/email/templates/error.txt
@@ -0,0 +1,11 @@
+{{ $start := mustGet (.State.GetCachedField 4) "Start" }}
+{{ $batchSize := mustGet (.State.GetCachedField 4) "BatchSize" }}
+{{ $discoveryTweets := mustGet (.State.GetCachedField 4) "DiscoveryTweets" }}
+{{ $phase := mustGet (.State.GetCachedField 4) "Phase" }}
+Robo-scout has encountered an error during the Scout procedure which started at {{ stamp $start }} with batchSize = {{ $batchSize }}, and discoveryTweets = {{ $discoveryTweets }}.
+
+The error occurred in the {{ $phase.String }} phase at {{ stamp .Time }}:
+"{{ .Error.Error }}"
+
+{{ if .State.Loaded }}Current state was loaded from disk:{{ else }}Current state was just created, so we are probably starting from scratch today:{{ end }}
+{{ .State.String }}
diff --git a/email/templates/finished.txt b/email/templates/finished.txt
@@ -0,0 +1,46 @@
+Robo-scout has finished the Scout procedure which started at {{ stamp .Started }} with batchSize = {{ .BatchSize }}, and discoveryTweets = {{ .DiscoveryTweets }}.
+
+This was achieved at {{ stamp .Finished }}, with a total time of {{ (duration .Finished .Started).String }}. The produced ScoutResult is as follows:
+- Discovery Phase
+  - DiscoveryStats.Developers = {{ .Result.DiscoveryStats.Developers }}
+  - DiscoveryStats.Games = {{ .Result.DiscoveryStats.Games }}
+  - DiscoveryStats.TweetsConsumed = {{ .Result.DiscoveryStats.TweetsConsumed }}
+  - DiscoveryStats.TotalSnapshots = {{ .Result.DiscoveryStats.TotalSnapshots }}
+  - DiscoveryStats.SnapshotsCreated = {{ .Result.DiscoveryStats.SnapshotsCreated }}
+- Update Phase
+  - UpdateStats.Developers = {{ .Result.UpdateStats.Developers }}
+  - UpdateStats.Games = {{ .Result.UpdateStats.Games }}
+  - UpdateStats.TweetsConsumed = {{ .Result.UpdateStats.TweetsConsumed }}
+  - UpdateStats.TotalSnapshots = {{ .Result.UpdateStats.TotalSnapshots }}
+  - UpdateStats.SnapshotsCreated = {{ .Result.UpdateStats.SnapshotsCreated }}
+- Snapshot Phase
+  - SnapshotStats.Developers = {{ .Result.SnapshotStats.Developers }}
+  - SnapshotStats.Games = {{ .Result.SnapshotStats.Games }}
+  - SnapshotStats.TweetsConsumed = {{ .Result.SnapshotStats.TweetsConsumed }}
+  - SnapshotStats.TotalSnapshots = {{ .Result.SnapshotStats.TotalSnapshots }}
+  - SnapshotStats.SnapshotsCreated = {{ .Result.SnapshotStats.SnapshotsCreated }}
+- Disable Phase
+  - DisableStats.EnabledDevelopersBefore = {{ .Result.DisableStats.EnabledDevelopersBefore }}
+  - DisableStats.DisabledDevelopersBefore = {{ .Result.DisableStats.DisabledDevelopersBefore }}
+  - DisableStats.EnabledDevelopersAfter = {{ .Result.DisableStats.EnabledDevelopersAfter }}
+  - DisableStats.DisabledDevelopersAfter = {{ .Result.DisableStats.DisabledDevelopersAfter }}
+  - DisableStats.DeletedDevelopers = {{ .Result.DisableStats.DeletedDevelopers }}
+  - DisableStats.TotalSampledDevelopers = {{ .Result.DisableStats.TotalSampledDevelopers }}
+- Enable Phase
+  - EnableStats.EnabledDevelopersBefore = {{ .Result.EnableStats.EnabledDevelopersBefore }}
+  - EnableStats.DisabledDevelopersBefore = {{ .Result.EnableStats.DisabledDevelopersBefore }}
+  - EnableStats.EnabledDevelopersAfter = {{ .Result.EnableStats.EnabledDevelopersAfter }}
+  - EnableStats.DisabledDevelopersAfter = {{ .Result.EnableStats.DisabledDevelopersAfter }}
+  - EnableStats.DeletedDevelopers = {{ .Result.EnableStats.DeletedDevelopers }}
+  - EnableStats.TotalSampledDevelopers = {{ .Result.EnableStats.TotalSampledDevelopers }}
+- Delete Phase
+  - DeleteStats.EnabledDevelopersBefore = {{ .Result.DeleteStats.EnabledDevelopersBefore }}
+  - DeleteStats.DisabledDevelopersBefore = {{ .Result.DeleteStats.DisabledDevelopersBefore }}
+  - DeleteStats.EnabledDevelopersAfter = {{ .Result.DeleteStats.EnabledDevelopersAfter }}
+  - DeleteStats.DisabledDevelopersAfter = {{ .Result.DeleteStats.DisabledDevelopersAfter }}
+  - DeleteStats.DeletedDevelopers = {{ .Result.DeleteStats.DeletedDevelopers }}
+  - DeleteStats.TotalSampledDevelopers = {{ .Result.DeleteStats.TotalSampledDevelopers }}
+- Measure Phase
+  - MeasureStats.SampledTrendingDevelopers = {{ .Result.MeasureStats.SampledTrendingDevelopers }}
+  - MeasureStats.EmailSendTimeTaken = {{ .Result.MeasureStats.EmailSendTimeTaken.String }}
+  - MeasureStats.EmailSize = {{ .Result.MeasureStats.EmailSize }}
diff --git a/main.go b/main.go
@@ -84,20 +84,6 @@ func main() {
 	}
 	log.INFO.Printf("Done setting up additional tasks in %s", time.Now().UTC().Sub(start).String())
 
-	start = time.Now().UTC()
-	log.INFO.Printf("Registering additional periodic tasks:")
-	for i, t := range []struct {
-		spec string
-		name string
-		args []any
-	}{
-		{"0 10 * * *", "scout", []any{100, 30250}},
-	} {
-		log.INFO.Printf("\tRegistering periodic task no. %d: \"%s\" for %q with args: %v", i+1, t.name, t.spec, t.args)
-		task.RegisterPeriodicTask(t.spec, t.name, t.args...)
-	}
-	log.INFO.Printf("Done setting up additional periodic tasks in %s", time.Now().UTC().Sub(start).String())
-
 	// Set the CLI app commands
 	cliApp.Commands = []cli.Command{
 		{
@@ -671,7 +657,70 @@ func main() {
 				case email.Started:
 					context = &StartedContext{state}
 				case email.Error:
-					break
+					context = &ErrorContext{
+						Time:  time.Now(),
+						Error: errors.New("this is a made-up error"),
+						State: state,
+					}
+				case email.Finished:
+					context = &email.FinishedContext{
+						BatchSize:       100,
+						DiscoveryTweets: 30250,
+						Started:         time.Now().Add(-1 * time.Hour * 3),
+						Finished:        time.Now(),
+						Result: &models.ScoutResult{
+							DiscoveryStats: &models.DiscoveryUpdateSnapshotStats{
+								Developers:       rand.Int63(),
+								Games:            rand.Int63(),
+								TweetsConsumed:   rand.Int63(),
+								TotalSnapshots:   rand.Int63(),
+								SnapshotsCreated: rand.Int63(),
+							},
+							UpdateStats: &models.DiscoveryUpdateSnapshotStats{
+								Developers:       rand.Int63(),
+								Games:            rand.Int63(),
+								TweetsConsumed:   rand.Int63(),
+								TotalSnapshots:   rand.Int63(),
+								SnapshotsCreated: rand.Int63(),
+							},
+							SnapshotStats: &models.DiscoveryUpdateSnapshotStats{
+								Developers:       rand.Int63(),
+								Games:            rand.Int63(),
+								TweetsConsumed:   rand.Int63(),
+								TotalSnapshots:   rand.Int63(),
+								SnapshotsCreated: rand.Int63(),
+							},
+							DisableStats: &models.DisableEnableDeleteStats{
+								EnabledDevelopersBefore:  rand.Int63(),
+								DisabledDevelopersBefore: rand.Int63(),
+								EnabledDevelopersAfter:   rand.Int63(),
+								DisabledDevelopersAfter:  rand.Int63(),
+								DeletedDevelopers:        rand.Int63(),
+								TotalSampledDevelopers:   rand.Int63(),
+							},
+							EnableStats: &models.DisableEnableDeleteStats{
+								EnabledDevelopersBefore:  rand.Int63(),
+								DisabledDevelopersBefore: rand.Int63(),
+								EnabledDevelopersAfter:   rand.Int63(),
+								DisabledDevelopersAfter:  rand.Int63(),
+								DeletedDevelopers:        rand.Int63(),
+								TotalSampledDevelopers:   rand.Int63(),
+							},
+							DeleteStats: &models.DisableEnableDeleteStats{
+								EnabledDevelopersBefore:  rand.Int63(),
+								DisabledDevelopersBefore: rand.Int63(),
+								EnabledDevelopersAfter:   rand.Int63(),
+								DisabledDevelopersAfter:  rand.Int63(),
+								DeletedDevelopers:        rand.Int63(),
+								TotalSampledDevelopers:   rand.Int63(),
+							},
+							MeasureStats: &models.MeasureStats{
+								SampledTrendingDevelopers: rand.Int63(),
+								EmailSendTimeTaken:        time.Second * 30,
+								EmailSize:                 rand.Int63(),
+							},
+						},
+					}
 				}
 
 				var executed *email.Template