diff --git a/src/dashboard/dashboard.go b/src/dashboard/dashboard.go index 1be7111..5f941dd 100644 --- a/src/dashboard/dashboard.go +++ b/src/dashboard/dashboard.go @@ -5,9 +5,11 @@ import ( "encoding/json" "fmt" "html/template" + "io" "io/fs" "net/http" "unbewohnte/wecr/config" + "unbewohnte/wecr/logger" "unbewohnte/wecr/worker" ) @@ -27,6 +29,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti mux := http.NewServeMux() res, err := fs.Sub(resFS, "res") if err != nil { + logger.Error("Failed to Sub embedded dashboard FS: %s", err) return nil } @@ -34,6 +37,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { template, err := template.ParseFS(res, "*.html") if err != nil { + logger.Error("Failed to parse embedded dashboard FS: %s", err) return } @@ -44,6 +48,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti jsonStats, err := json.MarshalIndent(statistics, "", " ") if err != nil { http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError) + logger.Error("Failed to marshal stats to send to the dashboard: %s", err) return } w.Header().Add("Content-type", "application/json") @@ -51,13 +56,42 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti }) mux.HandleFunc("/conf", func(w http.ResponseWriter, req *http.Request) { - jsonConf, err := json.MarshalIndent(webConf, "", " ") - if err != nil { - http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError) - return + switch req.Method { + case http.MethodPost: + var newConfig config.Conf + + defer req.Body.Close() + newConfigData, err := io.ReadAll(req.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusInternalServerError) + logger.Error("Failed to read new configuration from dashboard request: %s", err) + return + } + err = json.Unmarshal(newConfigData, &newConfig) + if err != nil { + http.Error(w, "Failed to unmarshal new configuration", http.StatusInternalServerError) + logger.Error("Failed to unmarshal new configuration from dashboard UI: %s", err) + return + } + + // DO NOT blindly replace global configuration. Manually check and replace values + webConf.Search.IsRegexp = newConfig.Search.IsRegexp + if len(newConfig.Search.Query) != 0 { + webConf.Search.Query = newConfig.Search.Query + } + + webConf.Logging.OutputLogs = newConfig.Logging.OutputLogs + + default: + jsonConf, err := json.MarshalIndent(webConf, "", " ") + if err != nil { + http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError) + logger.Error("Failed to marshal current configuration to send to the dashboard UI: %s", err) + return + } + w.Header().Add("Content-type", "application/json") + w.Write(jsonConf) } - w.Header().Add("Content-type", "application/json") - w.Write(jsonConf) }) server := &http.Server{ diff --git a/src/dashboard/res/index.html b/src/dashboard/res/index.html index 69c711c..339b0ed 100644 --- a/src/dashboard/res/index.html +++ b/src/dashboard/res/index.html @@ -29,55 +29,126 @@

Dashboard

-

Statistics

-
-
    -
  1. -
    -
    Pages visited
    -
    - 0 -
  2. -
  3. -
    -
    Matches found
    -
    - 0 -
  4. -
  5. -
    -
    Pages saved
    -
    - 0 -
  6. -
  7. -
    -
    Start time
    -
    - 0 -
  8. -
  9. -
    -
    Stopped
    -
    - false -
  10. -
+
+ + +
+

Statistics

+
+
    +
  1. +
    +
    Pages visited
    +
    + 0 +
  2. +
  3. +
    +
    Matches found
    +
    + 0 +
  4. +
  5. +
    +
    Pages saved
    +
    + 0 +
  6. +
  7. +
    +
    Start time
    +
    + 0 +
  8. +
  9. +
    +
    Stopped
    +
    + false +
  10. +
+
- +
+ +
+

Configuration

+
+ Make runtime changes to configuration + + + + + + + + + + + + + +
KeyValue
Query + +
Is regexp + +
+ +
+ +
+ +

+        
diff --git a/src/main.go b/src/main.go index fccb0d6..9276b2f 100644 --- a/src/main.go +++ b/src/main.go @@ -40,7 +40,7 @@ import ( "unbewohnte/wecr/worker" ) -const version = "v0.3.0" +const version = "v0.3.1" const ( defaultConfigFile string = "conf.json" @@ -392,8 +392,9 @@ func main() { // form a worker pool workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{ - Requests: conf.Requests, - Save: conf.Save, + Search: &conf.Search, + Requests: &conf.Requests, + Save: &conf.Save, BlacklistedDomains: conf.BlacklistedDomains, AllowedDomains: conf.AllowedDomains, VisitQueue: worker.VisitQueue{ diff --git a/src/web/extentions.go b/src/web/extentions.go index deaf930..a5f71b2 100644 --- a/src/web/extentions.go +++ b/src/web/extentions.go @@ -119,7 +119,6 @@ var DocumentExtentions = []string{ ".pl", ".lua", ".kt", - ".js", ".rb", ".asm", ".rar", diff --git a/src/worker/worker.go b/src/worker/worker.go index beb9ae1..abc7f89 100644 --- a/src/worker/worker.go +++ b/src/worker/worker.go @@ -40,8 +40,9 @@ type VisitQueue struct { // Worker configuration type WorkerConf struct { - Requests config.Requests - Save config.Save + Search *config.Search + Requests *config.Requests + Save *config.Save BlacklistedDomains []string AllowedDomains []string VisitQueue VisitQueue @@ -236,7 +237,6 @@ func (w *Worker) Work() { // find links pageLinks := web.FindPageLinks(pageData, pageURL) - go func() { if job.Depth > 1 { // decrement depth and add new jobs @@ -249,7 +249,7 @@ func (w *Worker) Work() { if link != job.URL { err = queue.InsertNewJob(w.Conf.VisitQueue.VisitQueue, web.Job{ URL: link, - Search: job.Search, + Search: *w.Conf.Search, Depth: job.Depth, }) if err != nil { @@ -265,7 +265,7 @@ func (w *Worker) Work() { if link != job.URL { w.Jobs <- web.Job{ URL: link, - Search: job.Search, + Search: *w.Conf.Search, Depth: job.Depth, } }