Browse Source

Web dashboard: ability to change runtime query

master
parent
commit
e5af2939cc
  1. 34
      src/dashboard/dashboard.go
  2. 100
      src/dashboard/res/index.html
  3. 7
      src/main.go
  4. 1
      src/web/extentions.go
  5. 10
      src/worker/worker.go

34
src/dashboard/dashboard.go

@ -5,9 +5,11 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"html/template" "html/template"
"io"
"io/fs" "io/fs"
"net/http" "net/http"
"unbewohnte/wecr/config" "unbewohnte/wecr/config"
"unbewohnte/wecr/logger"
"unbewohnte/wecr/worker" "unbewohnte/wecr/worker"
) )
@ -27,6 +29,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
mux := http.NewServeMux() mux := http.NewServeMux()
res, err := fs.Sub(resFS, "res") res, err := fs.Sub(resFS, "res")
if err != nil { if err != nil {
logger.Error("Failed to Sub embedded dashboard FS: %s", err)
return nil return nil
} }
@ -34,6 +37,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
template, err := template.ParseFS(res, "*.html") template, err := template.ParseFS(res, "*.html")
if err != nil { if err != nil {
logger.Error("Failed to parse embedded dashboard FS: %s", err)
return return
} }
@ -44,6 +48,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
jsonStats, err := json.MarshalIndent(statistics, "", " ") jsonStats, err := json.MarshalIndent(statistics, "", " ")
if err != nil { if err != nil {
http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError) http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError)
logger.Error("Failed to marshal stats to send to the dashboard: %s", err)
return return
} }
w.Header().Add("Content-type", "application/json") w.Header().Add("Content-type", "application/json")
@ -51,13 +56,42 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
}) })
mux.HandleFunc("/conf", func(w http.ResponseWriter, req *http.Request) { mux.HandleFunc("/conf", func(w http.ResponseWriter, req *http.Request) {
switch req.Method {
case http.MethodPost:
var newConfig config.Conf
defer req.Body.Close()
newConfigData, err := io.ReadAll(req.Body)
if err != nil {
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
logger.Error("Failed to read new configuration from dashboard request: %s", err)
return
}
err = json.Unmarshal(newConfigData, &newConfig)
if err != nil {
http.Error(w, "Failed to unmarshal new configuration", http.StatusInternalServerError)
logger.Error("Failed to unmarshal new configuration from dashboard UI: %s", err)
return
}
// DO NOT blindly replace global configuration. Manually check and replace values
webConf.Search.IsRegexp = newConfig.Search.IsRegexp
if len(newConfig.Search.Query) != 0 {
webConf.Search.Query = newConfig.Search.Query
}
webConf.Logging.OutputLogs = newConfig.Logging.OutputLogs
default:
jsonConf, err := json.MarshalIndent(webConf, "", " ") jsonConf, err := json.MarshalIndent(webConf, "", " ")
if err != nil { if err != nil {
http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError) http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError)
logger.Error("Failed to marshal current configuration to send to the dashboard UI: %s", err)
return return
} }
w.Header().Add("Content-type", "application/json") w.Header().Add("Content-type", "application/json")
w.Write(jsonConf) w.Write(jsonConf)
}
}) })
server := &http.Server{ server := &http.Server{

100
src/dashboard/res/index.html

@ -29,6 +29,10 @@
<div class="container"> <div class="container">
<h1>Dashboard</h1> <h1>Dashboard</h1>
<div style="height: 3rem;"></div>
<div class="container">
<h2>Statistics</h2> <h2>Statistics</h2>
<div id="statistics"> <div id="statistics">
<ol class="list-group list-group-numbered"> <ol class="list-group list-group-numbered">
@ -64,20 +68,87 @@
</li> </li>
</ol> </ol>
</div> </div>
</div>
<div style="height: 3rem;"></div>
<div class="container">
<h2>Configuration</h2>
<div>
<b>Make runtime changes to configuration</b>
<table class="table table-borderless">
<tr>
<th>Key</th>
<th>Value</th>
</tr>
<tr>
<th>Query</th>
<th>
<input type="text" id="conf_query">
</th>
</tr>
<tr>
<th>Is regexp</th>
<th>
<input type="text" id="conf_is_regexp">
</th>
</tr>
</table>
<button class="btn btn-primary" id="config_apply_button">
Apply
</button>
</div>
<!-- <h2>Configuration</h2> <div style="height: 3rem;"></div>
<pre id="configuration"></pre> -->
<pre id="conf_output"></pre>
</div>
</div> </div>
</body> </body>
<script> <script>
window.onload = function () { window.onload = function () {
let confOutput = document.getElementById("configuration"); let confOutput = document.getElementById("conf_output");
let pagesVisitedOut = document.getElementById("pages_visited"); let pagesVisitedOut = document.getElementById("pages_visited");
let matchesFoundOut = document.getElementById("matches_found"); let matchesFoundOut = document.getElementById("matches_found");
let pagesSavedOut = document.getElementById("pages_saved"); let pagesSavedOut = document.getElementById("pages_saved");
let startTimeOut = document.getElementById("start_time_unix"); let startTimeOut = document.getElementById("start_time_unix");
let stoppedOut = document.getElementById("stopped"); let stoppedOut = document.getElementById("stopped");
let applyConfButton = document.getElementById("config_apply_button");
let confQuery = document.getElementById("conf_query");
let confIsRegexp = document.getElementById("conf_is_regexp");
applyConfButton.addEventListener("click", (event) => {
let query = String(confQuery.value);
if (confIsRegexp.value === "0") {
isRegexp = false;
} else if (confIsRegexp.value === "1") {
isRegexp = true;
};
if (confIsRegexp.value === "false") {
isRegexp = false;
} else if (confIsRegexp.value === "true") {
isRegexp = true;
};
let newConf = {
"search": {
"is_regexp": isRegexp,
"query": query,
},
};
console.log(newConf);
fetch("/conf", {
method: "POST",
headers: {
"Content-type": "application/json",
},
body: JSON.stringify(newConf),
});
});
const interval = setInterval(function () { const interval = setInterval(function () {
// update statistics // update statistics
@ -90,13 +161,22 @@
startTimeOut.innerText = new Date(1000 * statistics.start_time_unix); startTimeOut.innerText = new Date(1000 * statistics.start_time_unix);
stoppedOut.innerText = statistics.stopped; stoppedOut.innerText = statistics.stopped;
}); });
// // update config just in case // update config
// fetch("/conf") fetch("/conf")
// .then((response) => response.text()) .then((response) => response.text())
// .then((response_text) => JSON.parse(response_text)) .then((config) => {
// .then((config) => { // "print" whole configuration
// confOutput.innerText = "Configuration: \n" + JSON.stringify(config); confOutput.innerText = config;
// });
// update values in the change table if they're empty
let confJSON = JSON.parse(config);
if (confQuery.value == "") {
confQuery.value = confJSON.search.query;
}
if (confIsRegexp.value == "") {
confIsRegexp.value = confJSON.search.is_regexp;
}
});
}, 650); }, 650);
}(); }();
</script> </script>

7
src/main.go

@ -40,7 +40,7 @@ import (
"unbewohnte/wecr/worker" "unbewohnte/wecr/worker"
) )
const version = "v0.3.0" const version = "v0.3.1"
const ( const (
defaultConfigFile string = "conf.json" defaultConfigFile string = "conf.json"
@ -392,8 +392,9 @@ func main() {
// form a worker pool // form a worker pool
workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{ workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{
Requests: conf.Requests, Search: &conf.Search,
Save: conf.Save, Requests: &conf.Requests,
Save: &conf.Save,
BlacklistedDomains: conf.BlacklistedDomains, BlacklistedDomains: conf.BlacklistedDomains,
AllowedDomains: conf.AllowedDomains, AllowedDomains: conf.AllowedDomains,
VisitQueue: worker.VisitQueue{ VisitQueue: worker.VisitQueue{

1
src/web/extentions.go

@ -119,7 +119,6 @@ var DocumentExtentions = []string{
".pl", ".pl",
".lua", ".lua",
".kt", ".kt",
".js",
".rb", ".rb",
".asm", ".asm",
".rar", ".rar",

10
src/worker/worker.go

@ -40,8 +40,9 @@ type VisitQueue struct {
// Worker configuration // Worker configuration
type WorkerConf struct { type WorkerConf struct {
Requests config.Requests Search *config.Search
Save config.Save Requests *config.Requests
Save *config.Save
BlacklistedDomains []string BlacklistedDomains []string
AllowedDomains []string AllowedDomains []string
VisitQueue VisitQueue VisitQueue VisitQueue
@ -236,7 +237,6 @@ func (w *Worker) Work() {
// find links // find links
pageLinks := web.FindPageLinks(pageData, pageURL) pageLinks := web.FindPageLinks(pageData, pageURL)
go func() { go func() {
if job.Depth > 1 { if job.Depth > 1 {
// decrement depth and add new jobs // decrement depth and add new jobs
@ -249,7 +249,7 @@ func (w *Worker) Work() {
if link != job.URL { if link != job.URL {
err = queue.InsertNewJob(w.Conf.VisitQueue.VisitQueue, web.Job{ err = queue.InsertNewJob(w.Conf.VisitQueue.VisitQueue, web.Job{
URL: link, URL: link,
Search: job.Search, Search: *w.Conf.Search,
Depth: job.Depth, Depth: job.Depth,
}) })
if err != nil { if err != nil {
@ -265,7 +265,7 @@ func (w *Worker) Work() {
if link != job.URL { if link != job.URL {
w.Jobs <- web.Job{ w.Jobs <- web.Job{
URL: link, URL: link,
Search: job.Search, Search: *w.Conf.Search,
Depth: job.Depth, Depth: job.Depth,
} }
} }

Loading…
Cancel
Save