Adding dataset images

.dockerignore

@@ -22,3 +22,5 @@ LICENSE
 README.md
 tox.ini
 *.secret.env
+./data/
+node_modules/

Dockerfile

@@ -1,6 +1,8 @@
 FROM frictionlessdata/datapackage-pipelines:2.1.8
 
-RUN apk --update --no-cache add bash wget
+RUN apk --update --no-cache add bash wget nodejs npm
+RUN npm install -g npm@latest
+RUN cd /pipelines/ && npm install puppeteer
 
 COPY docker-dpp-run.sh /dpp/docker/run.sh
 

datapackage_pipelines_migdar/flows/dataset_assets.py

@@ -0,0 +1,29 @@
+import os
+import subprocess
+
+import dataflows as DF
+
+SCREENSHOT = os.path.join(os.path.dirname(__file__), 'node', 'screenshot.js')
+
+
+def do_screenshot():
+    def func(row):
+        doc_id = row['doc_id']
+        url = f'https://yodaat.org/card/{doc_id}'
+        outpath = os.path.join('data', os.path.dirname(doc_id))
+        os.makedirs(outpath, exist_ok=True)
+        outpath = os.path.join('data', doc_id + '.png')
+        print('running', doc_id)
+        subprocess.call(['node', SCREENSHOT, url, outpath, '.card'])
+    return func
+
+
+def flow(*_, path='data/datasets_in_es'):
+    return DF.Flow(
+        DF.load('{}/datapackage.json'.format(path)),
+        do_screenshot()
+    )
+
+
+if __name__ == '__main__':
+    flow(path='https://api.yodaat.org/data/datasets_in_es').process()

datapackage_pipelines_migdar/flows/node/.gitignore

@@ -0,0 +1 @@
+node_modules/

datapackage_pipelines_migdar/flows/node/screenshot.js

@@ -0,0 +1,19 @@
+const puppeteer = require('puppeteer');
+
+(async () => {
+    const [url, filename, selector] = process.argv.slice(2);
+
+    const browser = await puppeteer.launch();
+    const page = await browser.newPage();
+    page.setViewport({width: 1300, height: 1200});
+    await page.goto(url);
+    await page.waitForSelector(selector);
+
+    const rect = await page.evaluate(selector => {
+        const element = document.querySelector(selector);
+        const {x, y, width, height} = element.getBoundingClientRect();
+        return {x, y, width, height};
+    }, selector);
+    await page.screenshot({path: filename, clip: rect});
+    await browser.close();
+})();