Unbewohnte
/
wecr
1
1
Fork 0
Code Issues Pull Requests Projects Releases 8 Wiki Activity
Browse Source

Web dashboard: ability to change runtime query

master
Kasianov Nikolai Alekseevich 1 year ago
parent
6fab9031b1
commit
e5af2939cc
5 changed files with 173 additions and 59 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 46
      src/dashboard/dashboard.go
  2. 168
      src/dashboard/res/index.html
  3. 7
      src/main.go
  4. 1
      src/web/extentions.go
  5. 10
      src/worker/worker.go

46
src/dashboard/dashboard.go
Unescape View File

@ -5,9 +5,11 @@ import (
"encoding/json"
"fmt"
"html/template"
"io"
"io/fs"
"net/http"
"unbewohnte/wecr/config"
"unbewohnte/wecr/logger"
"unbewohnte/wecr/worker"
)
@ -27,6 +29,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
mux := http.NewServeMux()
res, err := fs.Sub(resFS, "res")
if err != nil {
logger.Error("Failed to Sub embedded dashboard FS: %s", err)
return nil
}
@ -34,6 +37,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
template, err := template.ParseFS(res, "*.html")
if err != nil {
logger.Error("Failed to parse embedded dashboard FS: %s", err)
return
}
@ -44,6 +48,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
jsonStats, err := json.MarshalIndent(statistics, "", " ")
if err != nil {
http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError)
logger.Error("Failed to marshal stats to send to the dashboard: %s", err)
return
}
w.Header().Add("Content-type", "application/json")
@ -51,13 +56,42 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
})
mux.HandleFunc("/conf", func(w http.ResponseWriter, req *http.Request) {
jsonConf, err := json.MarshalIndent(webConf, "", " ")
if err != nil {
http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError)
return
switch req.Method {
case http.MethodPost:
var newConfig config.Conf
defer req.Body.Close()
newConfigData, err := io.ReadAll(req.Body)
if err != nil {
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
logger.Error("Failed to read new configuration from dashboard request: %s", err)
return
}
err = json.Unmarshal(newConfigData, &newConfig)
if err != nil {
http.Error(w, "Failed to unmarshal new configuration", http.StatusInternalServerError)
logger.Error("Failed to unmarshal new configuration from dashboard UI: %s", err)
return
}
// DO NOT blindly replace global configuration. Manually check and replace values
webConf.Search.IsRegexp = newConfig.Search.IsRegexp
if len(newConfig.Search.Query) != 0 {
webConf.Search.Query = newConfig.Search.Query
}
webConf.Logging.OutputLogs = newConfig.Logging.OutputLogs
default:
jsonConf, err := json.MarshalIndent(webConf, "", " ")
if err != nil {
http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError)
logger.Error("Failed to marshal current configuration to send to the dashboard UI: %s", err)
return
}
w.Header().Add("Content-type", "application/json")
w.Write(jsonConf)
}
w.Header().Add("Content-type", "application/json")
w.Write(jsonConf)
})
server := &http.Server{

168
src/dashboard/res/index.html
Unescape View File

@ -29,55 +29,126 @@
<div class="container">
<h1>Dashboard</h1>
<h2>Statistics</h2>
<div id="statistics">
<ol class="list-group list-group-numbered">
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Pages visited</div>
</div>
<span class="badge bg-primary rounded-pill" id="pages_visited">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Matches found</div>
</div>
<span class="badge bg-primary rounded-pill" id="matches_found">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Pages saved</div>
</div>
<span class="badge bg-primary rounded-pill" id="pages_saved">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Start time</div>
</div>
<span class="badge bg-primary rounded-pill" id="start_time_unix">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Stopped</div>
</div>
<span class="badge bg-primary rounded-pill" id="stopped">false</span>
</li>
</ol>
<div style="height: 3rem;"></div>
<div class="container">
<h2>Statistics</h2>
<div id="statistics">
<ol class="list-group list-group-numbered">
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Pages visited</div>
</div>
<span class="badge bg-primary rounded-pill" id="pages_visited">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Matches found</div>
</div>
<span class="badge bg-primary rounded-pill" id="matches_found">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Pages saved</div>
</div>
<span class="badge bg-primary rounded-pill" id="pages_saved">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Start time</div>
</div>
<span class="badge bg-primary rounded-pill" id="start_time_unix">0</span>
</li>
<li class="list-group-item d-flex justify-content-between align-items-start">
<div class="ms-2 me-auto">
<div class="fw-bold">Stopped</div>
</div>
<span class="badge bg-primary rounded-pill" id="stopped">false</span>
</li>
</ol>
</div>
</div>
<!-- <h2>Configuration</h2>
<pre id="configuration"></pre> -->
<div style="height: 3rem;"></div>
<div class="container">
<h2>Configuration</h2>
<div>
<b>Make runtime changes to configuration</b>
<table class="table table-borderless">
<tr>
<th>Key</th>
<th>Value</th>
</tr>
<tr>
<th>Query</th>
<th>
<input type="text" id="conf_query">
</th>
</tr>
<tr>
<th>Is regexp</th>
<th>
<input type="text" id="conf_is_regexp">
</th>
</tr>
</table>
<button class="btn btn-primary" id="config_apply_button">
Apply
</button>
</div>
<div style="height: 3rem;"></div>
<pre id="conf_output"></pre>
</div>
</div>
</body>
<script>
window.onload = function () {
let confOutput = document.getElementById("configuration");
let confOutput = document.getElementById("conf_output");
let pagesVisitedOut = document.getElementById("pages_visited");
let matchesFoundOut = document.getElementById("matches_found");
let pagesSavedOut = document.getElementById("pages_saved");
let startTimeOut = document.getElementById("start_time_unix");
let stoppedOut = document.getElementById("stopped");
let applyConfButton = document.getElementById("config_apply_button");
let confQuery = document.getElementById("conf_query");
let confIsRegexp = document.getElementById("conf_is_regexp");
applyConfButton.addEventListener("click", (event) => {
let query = String(confQuery.value);
if (confIsRegexp.value === "0") {
isRegexp = false;
} else if (confIsRegexp.value === "1") {
isRegexp = true;
};
if (confIsRegexp.value === "false") {
isRegexp = false;
} else if (confIsRegexp.value === "true") {
isRegexp = true;
};
let newConf = {
"search": {
"is_regexp": isRegexp,
"query": query,
},
};
console.log(newConf);
fetch("/conf", {
method: "POST",
headers: {
"Content-type": "application/json",
},
body: JSON.stringify(newConf),
});
});
const interval = setInterval(function () {
// update statistics
@ -90,13 +161,22 @@
startTimeOut.innerText = new Date(1000 * statistics.start_time_unix);
stoppedOut.innerText = statistics.stopped;
});
// // update config just in case
// fetch("/conf")
// .then((response) => response.text())
// .then((response_text) => JSON.parse(response_text))
// .then((config) => {
// confOutput.innerText = "Configuration: \n" + JSON.stringify(config);
// });
// update config
fetch("/conf")
.then((response) => response.text())
.then((config) => {
// "print" whole configuration
confOutput.innerText = config;
// update values in the change table if they're empty
let confJSON = JSON.parse(config);
if (confQuery.value == "") {
confQuery.value = confJSON.search.query;
}
if (confIsRegexp.value == "") {
confIsRegexp.value = confJSON.search.is_regexp;
}
});
}, 650);
}();
</script>

7
src/main.go
Unescape View File

@ -40,7 +40,7 @@ import (
"unbewohnte/wecr/worker"
)
const version = "v0.3.0"
const version = "v0.3.1"
const (
defaultConfigFile string = "conf.json"
@ -392,8 +392,9 @@ func main() {
// form a worker pool
workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{
Requests: conf.Requests,
Save: conf.Save,
Search: &conf.Search,
Requests: &conf.Requests,
Save: &conf.Save,
BlacklistedDomains: conf.BlacklistedDomains,
AllowedDomains: conf.AllowedDomains,
VisitQueue: worker.VisitQueue{

1
src/web/extentions.go
Unescape View File

@ -119,7 +119,6 @@ var DocumentExtentions = []string{
".pl",
".lua",
".kt",
".js",
".rb",
".asm",
".rar",

10
src/worker/worker.go
Unescape View File

@ -40,8 +40,9 @@ type VisitQueue struct {
// Worker configuration
type WorkerConf struct {
Requests config.Requests
Save config.Save
Search *config.Search
Requests *config.Requests
Save *config.Save
BlacklistedDomains []string
AllowedDomains []string
VisitQueue VisitQueue
@ -236,7 +237,6 @@ func (w *Worker) Work() {
// find links
pageLinks := web.FindPageLinks(pageData, pageURL)
go func() {
if job.Depth > 1 {
// decrement depth and add new jobs
@ -249,7 +249,7 @@ func (w *Worker) Work() {
if link != job.URL {
err = queue.InsertNewJob(w.Conf.VisitQueue.VisitQueue, web.Job{
URL: link,
Search: job.Search,
Search: *w.Conf.Search,
Depth: job.Depth,
})
if err != nil {
@ -265,7 +265,7 @@ func (w *Worker) Work() {
if link != job.URL {
w.Jobs <- web.Job{
URL: link,
Search: job.Search,
Search: *w.Conf.Search,
Depth: job.Depth,
}
}

Write Preview
Loading…
Cancel
Save