sync-package-db

#!/usr/bin/python
from gevent import spawn, monkey
monkey.patch_all()

import os
import sys
from queue import Queue
import json
import requests


URL = "https://www.archlinux.org/packages/search/json/?q=&repo=Core&repo=Extra&repo=Community&repo=Multilib&page="
#URL = "https://www.archlinux.org/packages/search/json/?q=&repo=Community-Testing&page="
PACKAGE_DB = {}


def worker(q):
    while True:
        page = q.get()
        print("Loading page", page)
        data = requests.get(URL + str(page)).json()
        for pkg in data["results"]:
            PACKAGE_DB[pkg["pkgname"]] = pkg
        print("Loaded page", page, "Current loaded packages:", len(PACKAGE_DB))
        q.task_done()


q = Queue()
[spawn(worker, q) for _ in range(10)]


for i in range(1, 50):
    q.put(i)


q.join()


print(len(PACKAGE_DB), "packaged loaded.\nWriting result")
with open(os.path.join(sys.path[0], "package.db"), "w") as f:
    json.dump(PACKAGE_DB, f)