Merge pull request #121 from creekorful/develop

Release 0.10.0
creekorful · Jan 6, 2021 · 4075dfc · 4075dfc
2 parents 6c4ecc1 + 829afcb
commit 4075dfc
Show file tree

Hide file tree

Showing 42 changed files with 989 additions and 1,878 deletions.
diff --git a/README.md b/README.md
@@ -30,50 +30,16 @@ and wait for all containers to start.
 
 # How to initiate crawling
 
-Since the API is exposed on localhost:15005, one can use it to start crawling:
+One can use the RabbitMQ dashhboard available at localhost:15003, and publish a new JSON object in the **crawlingQueue**.
 
-using trandoshanctl executable:
+The object should look like this:
 
-```sh
-$ trandoshanctl --api-token <token> schedule https://www.facebookcorewwwi.onion
-```
-
-or using the docker image:
-
-```sh
-$ docker run creekorful/trandoshanctl --api-token <token> --api-uri <uri> schedule https://www.facebookcorewwwi.onion
-```
-
-(you'll need to specify the api uri if you use the docker container)
-
-this will schedule given URL for crawling.
-
-## Example token
-
-Here's a working API token that you can use with trandoshanctl if you haven't changed the API signing key:
-
-```
-eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6InRyYW5kb3NoYW5jdGwiLCJyaWdodHMiOnsiUE9TVCI6WyIvdjEvdXJscyJdLCJHRVQiOlsiL3YxL3Jlc291cmNlcyJdfX0.jGA8WODYKtKy7ZijngoV8C3iWi1eTvMitA8Z1Is2GUg 
-```
-
-This token is the representation of the following payload:
-
-```
+```json
 {
-  "username": "trandoshanctl",
-  "rights": {
-    "POST": [
-      "/v1/urls"
-    ],
-    "GET": [
-      "/v1/resources"
-    ]
-  }
+  "url": "https://facebookcorewwwi.onion"
 }
 ```
 
-you may create your own tokens with the rights needed. In the future a CLI tool will allow token generation easily.
-
 ## How to speed up crawling
 
 If one want to speed up the crawling, he can scale the instance of crawling component in order to increase performances.
@@ -87,14 +53,6 @@ this will set the number of crawler instance to 5.
 
 # How to view results
 
-## Using trandoshanctl
-
-```sh
-$ trandoshanctl search <term>
-```
-
-## Using kibana
-
 You can use the Kibana dashboard available at http://localhost:15004. You will need to create an index pattern named '
 resources', and when it asks for the time field, choose 'time'.
 

diff --git a/build/docker/Dockerfile.tdsh-archiver b/build/docker/Dockerfile.tdsh-archiver
diff --git a/build/docker/Dockerfile.trandoshanctl b/build/docker/Dockerfile.trandoshanctl
diff --git a/cmd/tdsh-archiver/tdsh-archiver.go b/cmd/tdsh-archiver/tdsh-archiver.go
diff --git a/cmd/trandoshanctl/trandoshanctl.go b/cmd/trandoshanctl/trandoshanctl.go
diff --git a/deployments/docker/docker-compose.yml b/deployments/docker/docker-compose.yml
@@ -45,46 +45,47 @@ services:
       --log-level debug
       --hub-uri amqp://guest:guest@rabbitmq:5672
       --config-api-uri http://configapi:8080
+      --redis-uri redis:6379
     restart: always
     depends_on:
       - rabbitmq
-  archiver:
-    image: creekorful/tdsh-archiver:latest
+  indexer-local:
+    image: creekorful/tdsh-indexer:latest
     command: >
       --log-level debug
       --hub-uri amqp://guest:guest@rabbitmq:5672
-      --storage-dir /archive
+      --config-api-uri http://configapi:8080
+      --index-driver local
+      --index-dest /archive
     restart: always
     volumes:
       - archiverdata:/archive
     depends_on:
       - rabbitmq
-  indexer:
+      - configapi
+  indexer-es:
     image: creekorful/tdsh-indexer:latest
     command: >
       --log-level debug
       --hub-uri amqp://guest:guest@rabbitmq:5672
-      --elasticsearch-uri http://elasticsearch:9200
-      --signing-key K==M5RsU_DQa4_XSbkX?L27s^xWmde25
       --config-api-uri http://configapi:8080
-      --redis-uri redis:6379
+      --index-driver elastic
+      --index-dest http://elasticsearch:9200
     restart: always
     depends_on:
       - rabbitmq
       - elasticsearch
       - configapi
-      - redis
-    ports:
-      - 15005:8080
   configapi:
     image: creekorful/tdsh-configapi:latest
     command: >
       --log-level debug
       --hub-uri amqp://guest:guest@rabbitmq:5672
       --redis-uri redis:6379
-      --default-value forbidden-hostnames="[{\"hostname\": \"facebookcorewwwi.onion\"}, {\"hostname\": \"nytimes3xbfgragh.onion\"}]"
+      --default-value forbidden-hostnames="[]"
       --default-value allowed-mime-types="[{\"content-type\":\"text/\",\"extensions\":[\"html\",\"php\",\"aspx\", \"htm\"]}]"
       --default-value refresh-delay="{\"delay\": -1}"
+      --default-value blacklist-threshold="{\"threshold\": 5}"
     restart: always
     depends_on:
       - rabbitmq
@@ -97,10 +98,14 @@ services:
       --log-level debug
       --hub-uri amqp://guest:guest@rabbitmq:5672
       --config-api-uri http://configapi:8080
+      --redis-uri redis:6379
+      --tor-uri torproxy:9050
     restart: always
     depends_on:
       - rabbitmq
       - configapi
+      - redis
+      - torproxy
 
 volumes:
   esdata:
@@ -110,4 +115,4 @@ volumes:
   archiverdata:
     driver: local
   redisdata:
-    driver: local
+    driver: local
diff --git a/docs/architecture.png b/docs/architecture.png
diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go
diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go
diff --git a/internal/archiver/storage/storage.go b/internal/archiver/storage/storage.go