Create-claims-augmented-for-browser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f3e65b4f-591e-4a5d-b80b-36de29d3cd73",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "from kgtk.configure_kgtk_notebooks import ConfigureKGTK\n",
    "from kgtk.functions import kgtk, kypher\n",
    "from kgtk.io.kgtkreader import KgtkReader\n",
    "from kgtk.io.kgtkwriter import KgtkWriter\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7945ff66-c496-44c0-8f48-35695f4ca106",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "\n",
    "input_path = \"/data/amandeep/wikidata-20211027-dwd-v3\"\n",
    "output_path = \"/data/amandeep/wikidata-20211027-dwd-v3\"\n",
    "kgtk_path = \"/Users/amandeep/Github/kgtk\"\n",
    "\n",
    "graph_cache_path = None\n",
    "\n",
    "\n",
    "project_name = \"browser-claims-file\"\n",
    "files = 'claims'\n",
    "debug=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9b48f39a-b53f-4e82-af77-519b2d6e6edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "files = files.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d70bd4b7-e679-43b7-8512-8f8052822857",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User home: /nas/home/amandeep\n",
      "Current dir: /data/amandeep/github/kgtk-browser\n",
      "KGTK dir: /Users/amandeep/Github/kgtk\n",
      "Use-cases dir: /Users/amandeep/Github/kgtk/use-cases\n"
     ]
    }
   ],
   "source": [
    "ck = ConfigureKGTK(files, kgtk_path=kgtk_path)\n",
    "ck.configure_kgtk(input_graph_path=input_path,\n",
    "                  output_path=output_path,\n",
    "                  project_name=project_name,\n",
    "                 graph_cache_path=graph_cache_path,\n",
    "                 debug=debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dbdaab6b-1524-49a7-9212-7eac023e3581",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GRAPH: /data/amandeep/wikidata-20211027-dwd-v3\n",
      "KGTK_LABEL_FILE: /data/amandeep/wikidata-20211027-dwd-v3/labels.en.tsv.gz\n",
      "kypher: kgtk --debug query --graph-cache /data/amandeep/wikidata-20211027-dwd-v3/browser-claims-file/temp.browser-claims-file/wikidata.sqlite3.db\n",
      "STORE: /data/amandeep/wikidata-20211027-dwd-v3/browser-claims-file/temp.browser-claims-file/wikidata.sqlite3.db\n",
      "KGTK_GRAPH_CACHE: /data/amandeep/wikidata-20211027-dwd-v3/browser-claims-file/temp.browser-claims-file/wikidata.sqlite3.db\n",
      "TEMP: /data/amandeep/wikidata-20211027-dwd-v3/browser-claims-file/temp.browser-claims-file\n",
      "OUT: /data/amandeep/wikidata-20211027-dwd-v3/browser-claims-file\n",
      "USE_CASES_DIR: /Users/amandeep/Github/kgtk/use-cases\n",
      "EXAMPLES_DIR: /Users/amandeep/Github/kgtk/examples\n",
      "kgtk: kgtk --debug\n",
      "KGTK_OPTION_DEBUG: false\n",
      "claims: /data/amandeep/wikidata-20211027-dwd-v3/claims.tsv.gz\n"
     ]
    }
   ],
   "source": [
    "ck.print_env_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cc6770cb-5592-47d3-9e32-a0cd31d4759d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def separate_edges_and_qualifiers(input_file, output_edge_file, output_qualifier_file):\n",
    "    kr: KgtkReader = KgtkReader.open(Path(input_file),\n",
    "                                         error_file=sys.stderr)\n",
    "    ids = set()\n",
    "    for row in kr:\n",
    "        ids.add(row[kr.id_column_idx])\n",
    "    kr.close()\n",
    "    \n",
    "    kr: KgtkReader = KgtkReader.open(Path(input_file),\n",
    "                                         error_file=sys.stderr)\n",
    "    \n",
    "    \n",
    "    kw_edge: KgtkWriter = KgtkWriter.open(file_path=Path(output_edge_file),\n",
    "                                          error_file=sys.stderr,\n",
    "                                          column_names=kr.column_names,\n",
    "                                          mode=KgtkWriter.Mode.EDGE,\n",
    "                                          no_header=False)\n",
    "    kw_qualifier: KgtkWriter = KgtkWriter.open(file_path=Path(output_qualifier_file),\n",
    "                                          error_file=sys.stderr,\n",
    "                                          column_names=kr.column_names,\n",
    "                                          mode=KgtkWriter.Mode.EDGE,\n",
    "                                          no_header=False)\n",
    "    for row in kr:\n",
    "        id = row[kr.id_column_idx]\n",
    "        node1 = row[kr.node1_column_idx]\n",
    "        if node1 in ids:\n",
    "            kw_qualifier.write(row)\n",
    "        else:\n",
    "            kw_edge.write(row)\n",
    "    \n",
    "    kr.close()\n",
    "    kw_edge.close()\n",
    "    kw_qualifier.close()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c175a517-2fe6-4b85-821e-83be88e44731",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 11367  100 11367    0     0  26973      0 --:--:-- --:--:-- --:--:-- 26936\n"
     ]
    }
   ],
   "source": [
    "!curl https://raw.githubusercontent.com/usc-isi-i2/kgtk/dev/kgtk-properties/kgtk.properties.tsv -o $TEMP/kgtk.properties.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "87496c70-1500-48bb-bdcf-605230eef6ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(\"\"\"filter -i $TEMP/kgtk.properties.tsv\n",
    "        -p ';label;'\n",
    "        -o $TEMP/kgtk.properties.labels.tsv\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aade813f-4394-48ba-8a1a-b73ce9a9f30d",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(\"\"\"filter -i $TEMP/kgtk.properties.tsv\n",
    "        -p ';alias;'\n",
    "        -o $TEMP/kgtk.properties.aliases.tsv\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "66034d7c-fe5c-4ce6-b877-b28205b255d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(\"\"\"filter -i $TEMP/kgtk.properties.tsv\n",
    "        -p ';description;'\n",
    "        -o $TEMP/kgtk.properties.descriptions.tsv\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d3c064dd-4c02-448e-8208-9a3923cbc0d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(\"\"\"filter -i $TEMP/kgtk.properties.tsv\n",
    "        -p ';datatype;'\n",
    "        -o $TEMP/kgtk.properties.datatypes.tsv\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "34f86c2d-7ce6-49df-8dd9-609bc6e281a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(\"\"\"filter -i $TEMP/kgtk.properties.tsv\n",
    "        -p ';label,alias,description,datatype;'\n",
    "        --invert\n",
    "        -o $TEMP/kgtk.properties.claims.tsv\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "48f623ba-d031-431c-8da9-e65de3650ba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(\"\"\"cat \n",
    "    -i $GRAPH/statistics.Pinstance_count.star.tsv.gz\n",
    "    -i $GRAPH/statistics.Pinstance_count.tsv.gz\n",
    "    -i $GRAPH/statistics.Psubclass_count.star.tsv.gz\n",
    "    -i $GRAPH/derived.class.P1963computed.count.tsv.gz\n",
    "    -i $GRAPH/derived.P1963computed.subclass.count.star.tsv.gz\n",
    "    -i $GRAPH/derived.Pproperty_domain.tsv.gz\n",
    "    -i $TEMP/kgtk.properties.claims.tsv\n",
    "    -o $TEMP/derived.claims.augmented.tsv.gz\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "76002326-e7fc-4fb9-971b-dfc6d050e861",
   "metadata": {},
   "outputs": [],
   "source": [
    "separate_edges_and_qualifiers(f\"{os.environ['TEMP']}/derived.claims.augmented.tsv.gz\",\n",
    "                              f\"{os.environ['TEMP']}/derived.claims.edges.tsv.gz\",\n",
    "                              f\"{os.environ['TEMP']}/derived.claims.qualifiers.tsv.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1290c1c8-9eee-4342-9564-37db89ed895a",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(f\"\"\"cat \n",
    "        -i $GRAPH/labels.en.tsv.gz\n",
    "        -i $TEMP/kgtk.properties.labels.tsv\n",
    "        / sort\n",
    "         --extra '--parallel 24 --buffer-size 30% --temporary-directory {os.environ['TEMP']}'\n",
    "        -o $OUT/labels.en.tsv.gz\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3d3b99eb-650d-4c18-a195-2eace862cf31",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(f\"\"\"cat \n",
    "        -i $GRAPH/aliases.en.tsv.gz\n",
    "        -i $TEMP/kgtk.properties.aliases.tsv\n",
    "        / sort\n",
    "         --extra '--parallel 24 --buffer-size 30% --temporary-directory {os.environ['TEMP']}'\n",
    "        -o $OUT/aliases.en.tsv.gz\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "228b82d7-3144-46dd-847d-56244effba58",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(f\"\"\"cat \n",
    "        -i $GRAPH/descriptions.en.tsv.gz\n",
    "        -i $TEMP/kgtk.properties.descriptions.tsv\n",
    "        / sort\n",
    "         --extra '--parallel 24 --buffer-size 30% --temporary-directory {os.environ['TEMP']}'\n",
    "        -o $OUT/descriptions.en.tsv.gz\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "012fec42-7c3b-412d-87a9-e3ba94e9d372",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(f\"\"\"cat \n",
    "        -i $GRAPH/metadata.property.datatypes.tsv.gz\n",
    "        -i $TEMP/kgtk.properties.datatypes.tsv\n",
    "        / sort\n",
    "         --extra '--parallel 24 --buffer-size 30% --temporary-directory {os.environ['TEMP']}'\n",
    "        -o $OUT/metadata.property.datatypes.tsv.gz\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bc8c9626-acdc-43e6-86c6-7a135d4f2c09",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(f\"\"\"cat\n",
    "        -i $TEMP/derived.claims.qualifiers.tsv.gz\n",
    "        -i $GRAPH/qualifiers.tsv.gz\n",
    "        / sort \n",
    "         --extra '--parallel 24 --buffer-size 30% --temporary-directory {os.environ['TEMP']}'\n",
    "        -o $OUT/qualifiers.tsv.gz\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "18694b19-314c-4e1c-b8cb-651b1a4c2268",
   "metadata": {},
   "outputs": [],
   "source": [
    "kgtk(f\"\"\"cat\n",
    "        -i $TEMP/derived.claims.edges.tsv.gz\n",
    "        -i $GRAPH/claims.tsv.gz\n",
    "        -i $GRAPH/downloaded.wikipedia.short_abstracts.tsv.gz\n",
    "        / sort \n",
    "         --extra '--parallel 24 --buffer-size 30% --temporary-directory {os.environ['TEMP']}'\n",
    "        -o $OUT/claims.tsv.gz\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb762b48-67f9-4bde-b36d-983ea8ad221a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kgtk-env",
   "language": "python",
   "name": "kgtk-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}