Unbewohnte
/
wecr
1
1
Fork 0
Code Issues Pull Requests Projects Releases 8 Wiki Activity
Browse Source

Web dashboard: ability to change runtime query

master
Kasianov Nikolai Alekseevich 2 years ago
parent
6fab9031b1
commit
e5af2939cc
5 changed files with 173 additions and 59 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 46
      src/dashboard/dashboard.go
  2. 168
      src/dashboard/res/index.html
  3. 7
      src/main.go
  4. 1
      src/web/extentions.go
  5. 10
      src/worker/worker.go

46
src/dashboard/dashboard.go
Unescape View File

@ -5,9 +5,11 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"html/template" "html/template"
"io"
"io/fs" "io/fs"
"net/http" "net/http"
"unbewohnte/wecr/config" "unbewohnte/wecr/config"
"unbewohnte/wecr/logger"
"unbewohnte/wecr/worker" "unbewohnte/wecr/worker"
) )
@ -27,6 +29,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
mux := http.NewServeMux() mux := http.NewServeMux()
res, err := fs.Sub(resFS, "res") res, err := fs.Sub(resFS, "res")
if err != nil { if err != nil {
logger.Error("Failed to Sub embedded dashboard FS: %s", err)
return nil return nil
} }
@ -34,6 +37,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
template, err := template.ParseFS(res, "*.html") template, err := template.ParseFS(res, "*.html")
if err != nil { if err != nil {
logger.Error("Failed to parse embedded dashboard FS: %s", err)
return return
} }
@ -44,6 +48,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
jsonStats, err := json.MarshalIndent(statistics, "", " ") jsonStats, err := json.MarshalIndent(statistics, "", " ")
if err != nil { if err != nil {
http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError) http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError)
logger.Error("Failed to marshal stats to send to the dashboard: %s", err)
return return
} }
w.Header().Add("Content-type", "application/json") w.Header().Add("Content-type", "application/json")
@ -51,13 +56,42 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
}) })
mux.HandleFunc("/conf", func(w http.ResponseWriter, req *http.Request) { mux.HandleFunc("/conf", func(w http.ResponseWriter, req *http.Request) {
jsonConf, err := json.MarshalIndent(webConf, "", " ") switch req.Method {
if err != nil { case http.MethodPost:
http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError) var newConfig config.Conf
return
defer req.Body.Close()
newConfigData, err := io.ReadAll(req.Body)
if err != nil {
http.Error(w, "Failed to read request body", http.StatusInternalServerError)
logger.Error("Failed to read new configuration from dashboard request: %s", err)
return
}
err = json.Unmarshal(newConfigData, &newConfig)
if err != nil {
http.Error(w, "Failed to unmarshal new configuration", http.StatusInternalServerError)
logger.Error("Failed to unmarshal new configuration from dashboard UI: %s", err)
return
}
// DO NOT blindly replace global configuration. Manually check and replace values
webConf.Search.IsRegexp = newConfig.Search.IsRegexp
if len(newConfig.Search.Query) != 0 {
webConf.Search.Query = newConfig.Search.Query
}
webConf.Logging.OutputLogs = newConfig.Logging.OutputLogs
default:
jsonConf, err := json.MarshalIndent(webConf, "", " ")
if err != nil {
http.Error(w, "Failed to marshal configuration", http.StatusInternalServerError)
logger.Error("Failed to marshal current configuration to send to the dashboard UI: %s", err)
return
}
w.Header().Add("Content-type", "application/json")
w.Write(jsonConf)
} }
w.Header().Add("Content-type", "application/json")
w.Write(jsonConf)
}) })
server := &http.Server{ server := &http.Server{

168
src/dashboard/res/index.html
Unescape View File

@ -29,55 +29,126 @@
<div class="container"> <div class="container">
<h1>Dashboard</h1> <h1>Dashboard</h1>
<h2>Statistics</h2> <div style="height: 3rem;"></div>
<div id="statistics">
<ol class="list-group list-group-numbered">
<li class="list-group-item d-flex justify-content-between align-items-start"> <div class="container">
<div class="ms-2 me-auto"> <h2>Statistics</h2>
<div class="fw-bold">Pages visited</div> <div id="statistics">
</div> <ol class="list-group list-group-numbered">
<span class="badge bg-primary rounded-pill" id="pages_visited">0</span> <li class="list-group-item d-flex justify-content-between align-items-start">
</li> <div class="ms-2 me-auto">
<li class="list-group-item d-flex justify-content-between align-items-start"> <div class="fw-bold">Pages visited</div>
<div class="ms-2 me-auto"> </div>
<div class="fw-bold">Matches found</div> <span class="badge bg-primary rounded-pill" id="pages_visited">0</span>
</div> </li>
<span class="badge bg-primary rounded-pill" id="matches_found">0</span> <li class="list-group-item d-flex justify-content-between align-items-start">
</li> <div class="ms-2 me-auto">
<li class="list-group-item d-flex justify-content-between align-items-start"> <div class="fw-bold">Matches found</div>
<div class="ms-2 me-auto"> </div>
<div class="fw-bold">Pages saved</div> <span class="badge bg-primary rounded-pill" id="matches_found">0</span>
</div> </li>
<span class="badge bg-primary rounded-pill" id="pages_saved">0</span> <li class="list-group-item d-flex justify-content-between align-items-start">
</li> <div class="ms-2 me-auto">
<li class="list-group-item d-flex justify-content-between align-items-start"> <div class="fw-bold">Pages saved</div>
<div class="ms-2 me-auto"> </div>
<div class="fw-bold">Start time</div> <span class="badge bg-primary rounded-pill" id="pages_saved">0</span>
</div> </li>
<span class="badge bg-primary rounded-pill" id="start_time_unix">0</span> <li class="list-group-item d-flex justify-content-between align-items-start">
</li> <div class="ms-2 me-auto">
<li class="list-group-item d-flex justify-content-between align-items-start"> <div class="fw-bold">Start time</div>
<div class="ms-2 me-auto"> </div>
<div class="fw-bold">Stopped</div> <span class="badge bg-primary rounded-pill" id="start_time_unix">0</span>
</div> </li>
<span class="badge bg-primary rounded-pill" id="stopped">false</span> <li class="list-group-item d-flex justify-content-between align-items-start">
</li> <div class="ms-2 me-auto">
</ol> <div class="fw-bold">Stopped</div>
</div>
<span class="badge bg-primary rounded-pill" id="stopped">false</span>
</li>
</ol>
</div>
</div> </div>
<!-- <h2>Configuration</h2> <div style="height: 3rem;"></div>
<pre id="configuration"></pre> -->
<div class="container">
<h2>Configuration</h2>
<div>
<b>Make runtime changes to configuration</b>
<table class="table table-borderless">
<tr>
<th>Key</th>
<th>Value</th>
</tr>
<tr>
<th>Query</th>
<th>
<input type="text" id="conf_query">
</th>
</tr>
<tr>
<th>Is regexp</th>
<th>
<input type="text" id="conf_is_regexp">
</th>
</tr>
</table>
<button class="btn btn-primary" id="config_apply_button">
Apply
</button>
</div>
<div style="height: 3rem;"></div>
<pre id="conf_output"></pre>
</div>
</div> </div>
</body> </body>
<script> <script>
window.onload = function () { window.onload = function () {
let confOutput = document.getElementById("configuration"); let confOutput = document.getElementById("conf_output");
let pagesVisitedOut = document.getElementById("pages_visited"); let pagesVisitedOut = document.getElementById("pages_visited");
let matchesFoundOut = document.getElementById("matches_found"); let matchesFoundOut = document.getElementById("matches_found");
let pagesSavedOut = document.getElementById("pages_saved"); let pagesSavedOut = document.getElementById("pages_saved");
let startTimeOut = document.getElementById("start_time_unix"); let startTimeOut = document.getElementById("start_time_unix");
let stoppedOut = document.getElementById("stopped"); let stoppedOut = document.getElementById("stopped");
let applyConfButton = document.getElementById("config_apply_button");
let confQuery = document.getElementById("conf_query");
let confIsRegexp = document.getElementById("conf_is_regexp");
applyConfButton.addEventListener("click", (event) => {
let query = String(confQuery.value);
if (confIsRegexp.value === "0") {
isRegexp = false;
} else if (confIsRegexp.value === "1") {
isRegexp = true;
};
if (confIsRegexp.value === "false") {
isRegexp = false;
} else if (confIsRegexp.value === "true") {
isRegexp = true;
};
let newConf = {
"search": {
"is_regexp": isRegexp,
"query": query,
},
};
console.log(newConf);
fetch("/conf", {
method: "POST",
headers: {
"Content-type": "application/json",
},
body: JSON.stringify(newConf),
});
});
const interval = setInterval(function () { const interval = setInterval(function () {
// update statistics // update statistics
@ -90,13 +161,22 @@
startTimeOut.innerText = new Date(1000 * statistics.start_time_unix); startTimeOut.innerText = new Date(1000 * statistics.start_time_unix);
stoppedOut.innerText = statistics.stopped; stoppedOut.innerText = statistics.stopped;
}); });
// // update config just in case // update config
// fetch("/conf") fetch("/conf")
// .then((response) => response.text()) .then((response) => response.text())
// .then((response_text) => JSON.parse(response_text)) .then((config) => {
// .then((config) => { // "print" whole configuration
// confOutput.innerText = "Configuration: \n" + JSON.stringify(config); confOutput.innerText = config;
// });
// update values in the change table if they're empty
let confJSON = JSON.parse(config);
if (confQuery.value == "") {
confQuery.value = confJSON.search.query;
}
if (confIsRegexp.value == "") {
confIsRegexp.value = confJSON.search.is_regexp;
}
});
}, 650); }, 650);
}(); }();
</script> </script>

7
src/main.go
Unescape View File

@ -40,7 +40,7 @@ import (
"unbewohnte/wecr/worker" "unbewohnte/wecr/worker"
) )
const version = "v0.3.0" const version = "v0.3.1"
const ( const (
defaultConfigFile string = "conf.json" defaultConfigFile string = "conf.json"
@ -392,8 +392,9 @@ func main() {
// form a worker pool // form a worker pool
workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{ workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{
Requests: conf.Requests, Search: &conf.Search,
Save: conf.Save, Requests: &conf.Requests,
Save: &conf.Save,
BlacklistedDomains: conf.BlacklistedDomains, BlacklistedDomains: conf.BlacklistedDomains,
AllowedDomains: conf.AllowedDomains, AllowedDomains: conf.AllowedDomains,
VisitQueue: worker.VisitQueue{ VisitQueue: worker.VisitQueue{

1
src/web/extentions.go
Unescape View File

@ -119,7 +119,6 @@ var DocumentExtentions = []string{
".pl", ".pl",
".lua", ".lua",
".kt", ".kt",
".js",
".rb", ".rb",
".asm", ".asm",
".rar", ".rar",

10
src/worker/worker.go
Unescape View File

@ -40,8 +40,9 @@ type VisitQueue struct {
// Worker configuration // Worker configuration
type WorkerConf struct { type WorkerConf struct {
Requests config.Requests Search *config.Search
Save config.Save Requests *config.Requests
Save *config.Save
BlacklistedDomains []string BlacklistedDomains []string
AllowedDomains []string AllowedDomains []string
VisitQueue VisitQueue VisitQueue VisitQueue
@ -236,7 +237,6 @@ func (w *Worker) Work() {
// find links // find links
pageLinks := web.FindPageLinks(pageData, pageURL) pageLinks := web.FindPageLinks(pageData, pageURL)
go func() { go func() {
if job.Depth > 1 { if job.Depth > 1 {
// decrement depth and add new jobs // decrement depth and add new jobs
@ -249,7 +249,7 @@ func (w *Worker) Work() {
if link != job.URL { if link != job.URL {
err = queue.InsertNewJob(w.Conf.VisitQueue.VisitQueue, web.Job{ err = queue.InsertNewJob(w.Conf.VisitQueue.VisitQueue, web.Job{
URL: link, URL: link,
Search: job.Search, Search: *w.Conf.Search,
Depth: job.Depth, Depth: job.Depth,
}) })
if err != nil { if err != nil {
@ -265,7 +265,7 @@ func (w *Worker) Work() {
if link != job.URL { if link != job.URL {
w.Jobs <- web.Job{ w.Jobs <- web.Job{
URL: link, URL: link,
Search: job.Search, Search: *w.Conf.Search,
Depth: job.Depth, Depth: job.Depth,
} }
} }

Write Preview
Loading…
Cancel
Save