From 8b8e2f40b762b8e866ca0ed2345997a0dd453704 Mon Sep 17 00:00:00 2001 From: Unbewohnte <65883674+Unbewohnte@users.noreply.github.com> Date: Sun, 25 Apr 2021 09:39:13 +0300 Subject: [PATCH] Implemented worker pool --- OBM.go | 103 ++++++++++++++++++++++----------------------- manager/helpers.go | 35 +++++++++++++-- 2 files changed, 82 insertions(+), 56 deletions(-) diff --git a/OBM.go b/OBM.go index 1b8146d..71de084 100644 --- a/OBM.go +++ b/OBM.go @@ -1,11 +1,7 @@ package main import ( - "errors" "fmt" - "image" - "image/color" - "image/png" "os" "path/filepath" "sync" @@ -20,41 +16,40 @@ var ( WG sync.WaitGroup ) -// creates a complete black image file -func createBlackBG(width, height int) error { - bgfile, err := os.Create("blackBG.png") - if err != nil { - return errors.New(fmt.Sprintf("Could not create black background file : %s", err)) - } - image := image.NewRGBA(image.Rect(0, 0, width, height)) - bounds := image.Bounds() +type job struct { + songPath string + pathToImage string +} - for y := 0; y < bounds.Max.Y; y++ { - for x := 0; x < bounds.Max.X; x++ { - image.Set(x, y, color.Black) +type result struct { + successful uint64 + failed uint64 +} + +// a basic implementation of a concurrent worker +func worker(jobs <-chan job, results chan result, WG *sync.WaitGroup) { + defer WG.Done() + for job := range jobs { + s, f := manager.ReplaceBackgrounds(job.songPath, job.pathToImage) + results <- result{ + successful: s, + failed: f, } } - err = png.Encode(bgfile, image) - if err != nil { - return errors.New(fmt.Sprintf("Could not encode an image : %s", err)) - } - err = bgfile.Close() - if err != nil { - return errors.New(fmt.Sprintf("Could not close the background file : %s", err)) - } - return nil } -// a basic implementation of a concurrent worker -func worker(paths <-chan string, replacementImage string, successful, failed *uint64, WG *sync.WaitGroup) { - defer WG.Done() - for songPath := range paths { - s, f := manager.ReplaceBackgrounds(songPath, replacementImage) - *successful += s - *failed += f +func workerPool(jobs chan job, results chan result, numOfWorkers int, WG *sync.WaitGroup) { + // check if there are less jobs than workers + if numOfWorkers > len(jobs) { + numOfWorkers = len(jobs) } + // replacing backgrounds for each beatmap concurrently + for i := 0; i < numOfWorkers; i++ { + WG.Add(1) + go worker(jobs, results, WG) + } } func init() { @@ -67,19 +62,19 @@ func init() { return } - // settings file does not exist, so create it and exit + // settings file does not exist, so create it and exit (assuming that this is the first run) settings.CreateSettingsFile() os.Exit(0) } func main() { - startingTime := time.Now().UTC() + startingTime := time.Now() settings := settings.GetSettings() - // process the given settings + // processing given settings if settings.CreateBlackBGImage { - err := createBlackBG(1920, 1080) + err := manager.CreateBlackBG(1920, 1080) if err == nil { logger.LogInfo("Successfully created black background") } else { @@ -102,32 +97,34 @@ func main() { logger.LogError(true, fmt.Sprintf("Error reading osu songs directory : %s", err.Error())) } - // storing all paths to each beatmap - songPaths := make(chan string, len(osuSongsDirContents)) + // creating jobs for workers + jobs := make(chan job, len(osuSongsDirContents)) for _, songDir := range osuSongsDirContents { if songDir.IsDir() { - songPaths <- filepath.Join(osuSongsDir, songDir.Name()) + jobs <- job{ + songPath: filepath.Join(osuSongsDir, songDir.Name()), + pathToImage: settings.ReplacementImagePath, + } } } - logger.LogInfo(fmt.Sprintf("Found %d song folders", len(songPaths))) - - // check if there is less job than workers - if settings.Workers > len(songPaths) { - settings.Workers = len(songPaths) - } + close(jobs) + logger.LogInfo(fmt.Sprintf("Found %d song folders", len(jobs))) - // replacing backgrounds for each beatmap concurrently - var successful, failed uint64 = 0, 0 - for i := 0; i < int(settings.Workers); i++ { - WG.Add(1) - go worker(songPaths, settings.ReplacementImagePath, &successful, &failed, &WG) - } + results := make(chan result, len(jobs)) + workerPool(jobs, results, settings.Workers, &WG) - close(songPaths) WG.Wait() + close(results) - endTime := time.Now().UTC() + // extracting results and logging the last message + var successful, failed uint64 = 0, 0 + for result := range results { + successful += result.successful + failed += result.failed + } + total := successful + failed - logger.LogInfo(fmt.Sprintf("\n\nDONE in %v . %d successful; %d failed", endTime.Sub(startingTime), successful, failed)) + logger.LogInfo(fmt.Sprintf("DONE in %v. %d successful (%d%%/100%%); %d failed (%d%%/100%%)", + time.Since(startingTime), successful, successful/total*100, failed, failed/total*100)) } diff --git a/manager/helpers.go b/manager/helpers.go index 170751d..f07316a 100644 --- a/manager/helpers.go +++ b/manager/helpers.go @@ -3,12 +3,41 @@ package manager import ( "errors" "fmt" + "image" + "image/color" + "image/png" "io" "os" "strings" ) -// checks if given string contains ".osu" file extention +// creates a complete black image file +func CreateBlackBG(width, height int) error { + bgfile, err := os.Create("blackBG.png") + if err != nil { + return errors.New(fmt.Sprintf("Could not create black background file : %s", err)) + } + image := image.NewRGBA(image.Rect(0, 0, width, height)) + bounds := image.Bounds() + + for y := 0; y < bounds.Max.Y; y++ { + for x := 0; x < bounds.Max.X; x++ { + image.Set(x, y, color.Black) + } + } + err = png.Encode(bgfile, image) + if err != nil { + return errors.New(fmt.Sprintf("Could not encode an image : %s", err)) + } + err = bgfile.Close() + if err != nil { + return errors.New(fmt.Sprintf("Could not close the background file : %s", err)) + } + + return nil +} + +// checks if given string contains ".osu" file extention (NOT EXPORTED !) func isBeatmap(filename string) bool { if len(filename) < 5 { return false @@ -19,7 +48,7 @@ func isBeatmap(filename string) bool { return false } -// checks if given string contains the image file extention +// checks if given string contains the image file extention (NOT EXPORTED !) func isImage(filename string) bool { var imageExtentions []string = []string{"jpeg", "jpg", "png", "JPEG", "JPG", "PNG"} for _, extention := range imageExtentions { @@ -30,7 +59,7 @@ func isImage(filename string) bool { return false } -// opens given files, copies one into another +// opens given files, copies one into another (NOT EXPORTED !) func copyFile(src, dst string) error { srcFile, err := os.Open(src) if err != nil {