Web dashboard: ability to change runtime query

src/dashboard/dashboard.go

@@ -5,9 +5,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"html/template"
+	"io"
 	"io/fs"
 	"net/http"
 	"unbewohnte/wecr/config"
+	"unbewohnte/wecr/logger"
 	"unbewohnte/wecr/worker"
 )
 
@@ -27,6 +29,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
 	mux := http.NewServeMux()
 	res, err := fs.Sub(resFS, "res")
 	if err != nil {
+		logger.Error("Failed to Sub embedded dashboard FS: %s", err)
 		return nil
 	}
 
@@ -34,6 +37,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
 	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
 		template, err := template.ParseFS(res, "*.html")
 		if err != nil {
+			logger.Error("Failed to parse embedded dashboard FS: %s", err)
 			return
 		}
 
@@ -44,6 +48,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
 		jsonStats, err := json.MarshalIndent(statistics, "", " ")
 		if err != nil {
 			http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError)
+			logger.Error("Failed to marshal stats to send to the dashboard: %s", err)
 			return
 		}
 		w.Header().Add("Content-type", "application/json")
@@ -51,13 +56,42 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
 	})
 
 	mux.HandleFunc("/conf", func(w http.ResponseWriter, req *http.Request) {
-		jsonConf, err := json.MarshalIndent(webConf, "", " ")
+		switch req.Method {
-		if err != nil {
+		case http.MethodPost:
-			http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError)
+			var newConfig config.Conf
-			return
+
+			defer req.Body.Close()
+			newConfigData, err := io.ReadAll(req.Body)
+			if err != nil {
+				http.Error(w, "Failed to read request body", http.StatusInternalServerError)
+				logger.Error("Failed to read new configuration from dashboard request: %s", err)
+				return
+			}
+			err = json.Unmarshal(newConfigData, &newConfig)
+			if err != nil {
+				http.Error(w, "Failed to unmarshal new configuration", http.StatusInternalServerError)
+				logger.Error("Failed to unmarshal new configuration from dashboard UI: %s", err)
+				return
+			}
+
+			// DO NOT blindly replace global configuration. Manually check and replace values
+			webConf.Search.IsRegexp = newConfig.Search.IsRegexp
+			if len(newConfig.Search.Query) != 0 {
+				webConf.Search.Query = newConfig.Search.Query
+			}
+
+			webConf.Logging.OutputLogs = newConfig.Logging.OutputLogs
+
+		default:
+			jsonConf, err := json.MarshalIndent(webConf, "", " ")
+			if err != nil {
+				http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError)
+				logger.Error("Failed to marshal current configuration to send to the dashboard UI: %s", err)
+				return
+			}
+			w.Header().Add("Content-type", "application/json")
+			w.Write(jsonConf)
 		}
-		w.Header().Add("Content-type", "application/json")
-		w.Write(jsonConf)
 	})
 
 	server := &http.Server{

src/dashboard/res/index.html

@@ -29,55 +29,126 @@
     <div class="container">
         <h1>Dashboard</h1>
 
-        <h2>Statistics</h2>
+        <div style="height: 3rem;"></div>
-        <div id="statistics">
+
-            <ol class="list-group list-group-numbered">
+
-                <li class="list-group-item d-flex justify-content-between align-items-start">
+        <div class="container">
-                    <div class="ms-2 me-auto">
+            <h2>Statistics</h2>
-                        <div class="fw-bold">Pages visited</div>
+            <div id="statistics">
-                    </div>
+                <ol class="list-group list-group-numbered">
-                    <span class="badge bg-primary rounded-pill" id="pages_visited">0</span>
+                    <li class="list-group-item d-flex justify-content-between align-items-start">
-                </li>
+                        <div class="ms-2 me-auto">
-                <li class="list-group-item d-flex justify-content-between align-items-start">
+                            <div class="fw-bold">Pages visited</div>
-                    <div class="ms-2 me-auto">
+                        </div>
-                        <div class="fw-bold">Matches found</div>
+                        <span class="badge bg-primary rounded-pill" id="pages_visited">0</span>
-                    </div>
+                    </li>
-                    <span class="badge bg-primary rounded-pill" id="matches_found">0</span>
+                    <li class="list-group-item d-flex justify-content-between align-items-start">
-                </li>
+                        <div class="ms-2 me-auto">
-                <li class="list-group-item d-flex justify-content-between align-items-start">
+                            <div class="fw-bold">Matches found</div>
-                    <div class="ms-2 me-auto">
+                        </div>
-                        <div class="fw-bold">Pages saved</div>
+                        <span class="badge bg-primary rounded-pill" id="matches_found">0</span>
-                    </div>
+                    </li>
-                    <span class="badge bg-primary rounded-pill" id="pages_saved">0</span>
+                    <li class="list-group-item d-flex justify-content-between align-items-start">
-                </li>
+                        <div class="ms-2 me-auto">
-                <li class="list-group-item d-flex justify-content-between align-items-start">
+                            <div class="fw-bold">Pages saved</div>
-                    <div class="ms-2 me-auto">
+                        </div>
-                        <div class="fw-bold">Start time</div>
+                        <span class="badge bg-primary rounded-pill" id="pages_saved">0</span>
-                    </div>
+                    </li>
-                    <span class="badge bg-primary rounded-pill" id="start_time_unix">0</span>
+                    <li class="list-group-item d-flex justify-content-between align-items-start">
-                </li>
+                        <div class="ms-2 me-auto">
-                <li class="list-group-item d-flex justify-content-between align-items-start">
+                            <div class="fw-bold">Start time</div>
-                    <div class="ms-2 me-auto">
+                        </div>
-                        <div class="fw-bold">Stopped</div>
+                        <span class="badge bg-primary rounded-pill" id="start_time_unix">0</span>
-                    </div>
+                    </li>
-                    <span class="badge bg-primary rounded-pill" id="stopped">false</span>
+                    <li class="list-group-item d-flex justify-content-between align-items-start">
-                </li>
+                        <div class="ms-2 me-auto">
-            </ol>
+                            <div class="fw-bold">Stopped</div>
+                        </div>
+                        <span class="badge bg-primary rounded-pill" id="stopped">false</span>
+                    </li>
+                </ol>
+            </div>
         </div>
 
-        <!-- <h2>Configuration</h2>
+        <div style="height: 3rem;"></div>
-        <pre id="configuration"></pre> -->
+
+        <div class="container">
+            <h2>Configuration</h2>
+            <div>
+                <b>Make runtime changes to configuration</b>
+                <table class="table table-borderless">
+                    <tr>
+                        <th>Key</th>
+                        <th>Value</th>
+                    </tr>
+                    <tr>
+                        <th>Query</th>
+                        <th>
+                            <input type="text" id="conf_query">
+                        </th>
+                    </tr>
+                    <tr>
+                        <th>Is regexp</th>
+                        <th>
+                            <input type="text" id="conf_is_regexp">
+                        </th>
+                    </tr>
+                </table>
+                <button class="btn btn-primary" id="config_apply_button">
+                    Apply
+                </button>
+            </div>
+
+            <div style="height: 3rem;"></div>
+
+            <pre id="conf_output"></pre>
+        </div>
     </div>
 </body>
 
 <script>
     window.onload = function () {
-        let confOutput = document.getElementById("configuration");
+        let confOutput = document.getElementById("conf_output");
         let pagesVisitedOut = document.getElementById("pages_visited");
         let matchesFoundOut = document.getElementById("matches_found");
         let pagesSavedOut = document.getElementById("pages_saved");
         let startTimeOut = document.getElementById("start_time_unix");
         let stoppedOut = document.getElementById("stopped");
+        let applyConfButton = document.getElementById("config_apply_button");
+        let confQuery = document.getElementById("conf_query");
+        let confIsRegexp = document.getElementById("conf_is_regexp");
+
+        applyConfButton.addEventListener("click", (event) => {
+            let query = String(confQuery.value);
+
+            if (confIsRegexp.value === "0") {
+                isRegexp = false;
+            } else if (confIsRegexp.value === "1") {
+                isRegexp = true;
+            };
+            if (confIsRegexp.value === "false") {
+                isRegexp = false;
+            } else if (confIsRegexp.value === "true") {
+                isRegexp = true;
+            };
+
+            let newConf = {
+                "search": {
+                    "is_regexp": isRegexp,
+                    "query": query,
+                },
+            };
+
+            console.log(newConf);
+
+            fetch("/conf", {
+                method: "POST",
+                headers: {
+                    "Content-type": "application/json",
+                },
+                body: JSON.stringify(newConf),
+            });
+        });
 
         const interval = setInterval(function () {
             // update statistics
@@ -90,13 +161,22 @@
                     startTimeOut.innerText = new Date(1000 * statistics.start_time_unix);
                     stoppedOut.innerText = statistics.stopped;
                 });
-            // // update config just in case
+            // update config
-            // fetch("/conf")
+            fetch("/conf")
-            //     .then((response) => response.text())
+                .then((response) => response.text())
-            //     .then((response_text) => JSON.parse(response_text))
+                .then((config) => {
-            //     .then((config) => {
+                    // "print" whole configuration
-            //         confOutput.innerText = "Configuration: \n" + JSON.stringify(config);
+                    confOutput.innerText = config;
-            //     });
+
+                    // update values in the change table if they're empty
+                    let confJSON = JSON.parse(config);
+                    if (confQuery.value == "") {
+                        confQuery.value = confJSON.search.query;
+                    }
+                    if (confIsRegexp.value == "") {
+                        confIsRegexp.value = confJSON.search.is_regexp;
+                    }
+                });
         }, 650);
     }();
 </script>

src/main.go

@@ -40,7 +40,7 @@ import (
 	"unbewohnte/wecr/worker"
 )
 
-const version = "v0.3.0"
+const version = "v0.3.1"
 
 const (
 	defaultConfigFile           string = "conf.json"
@@ -392,8 +392,9 @@ func main() {
 
 	// form a worker pool
 	workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{
-		Requests:           conf.Requests,
+		Search:             &conf.Search,
-		Save:               conf.Save,
+		Requests:           &conf.Requests,
+		Save:               &conf.Save,
 		BlacklistedDomains: conf.BlacklistedDomains,
 		AllowedDomains:     conf.AllowedDomains,
 		VisitQueue: worker.VisitQueue{

src/web/extentions.go

@@ -119,7 +119,6 @@ var DocumentExtentions = []string{
 	".pl",
 	".lua",
 	".kt",
-	".js",
 	".rb",
 	".asm",
 	".rar",

src/worker/worker.go

@@ -40,8 +40,9 @@ type VisitQueue struct {
 
 // Worker configuration
 type WorkerConf struct {
-	Requests           config.Requests
+	Search             *config.Search
-	Save               config.Save
+	Requests           *config.Requests
+	Save               *config.Save
 	BlacklistedDomains []string
 	AllowedDomains     []string
 	VisitQueue         VisitQueue
@@ -236,7 +237,6 @@ func (w *Worker) Work() {
 
 		// find links
 		pageLinks := web.FindPageLinks(pageData, pageURL)
-
 		go func() {
 			if job.Depth > 1 {
 				// decrement depth and add new jobs
@@ -249,7 +249,7 @@ func (w *Worker) Work() {
 						if link != job.URL {
 							err = queue.InsertNewJob(w.Conf.VisitQueue.VisitQueue, web.Job{
 								URL:    link,
-								Search: job.Search,
+								Search: *w.Conf.Search,
 								Depth:  job.Depth,
 							})
 							if err != nil {
@@ -265,7 +265,7 @@ func (w *Worker) Work() {
 						if link != job.URL {
 							w.Jobs <- web.Job{
 								URL:    link,
-								Search: job.Search,
+								Search: *w.Conf.Search,
 								Depth:  job.Depth,
 							}
 						}