extract_blocks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "358d1bfc-c698-4f8b-a24d-22c327e8b650",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from typing import NamedTuple, Optional\n",
    "import warnings\n",
    "\n",
    "CWD = Path.cwd()\n",
    "UCD_PATH = CWD / \"third_party/ucd\"\n",
    "ICON_PATH = CWD / \"icons/ucd\"\n",
    "\n",
    "\n",
    "def hex_format(n: int, /, digits: int = 4) -> str:\n",
    "    return f\"{n:0{digits}x}\".upper()\n",
    "\n",
    "\n",
    "def hex2int(s: str, /) -> int:\n",
    "    try:\n",
    "        return int(s, 16)\n",
    "    except ValueError as exc:\n",
    "        raise ValueError(f\"Invalid hex string: {s}\") from exc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67dc059a",
   "metadata": {},
   "source": [
    "## Metadata, source: <https://www.unicode.org/L2/L1999/UnicodeData.html>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ba7af85-676e-4c7c-9950-2e69b4bd42e3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "raw_metadata = [\n",
    "    (\"Code value\", \"normative\", \"Code value in 4-digit hexadecimal format.\"),\n",
    "    (\n",
    "        \"Character name\",\n",
    "        \"normative\",\n",
    "        \"These names match exactly the names published in Chapter 7 of the Unicode Standard, Version 2.0, except for the two additional characters.\",\n",
    "    ),\n",
    "    (\n",
    "        \"General category\",\n",
    "        \"normative / informative(see below)\",\n",
    "        'This is a useful breakdown into various \"character types\" which can be used as a default categorization in implementations. See below for a brief explanation.',\n",
    "    ),\n",
    "    (\n",
    "        \"Canonical combining classes\",\n",
    "        \"normative\",\n",
    "        \"The classes used for the Canonical Ordering Algorithm in the Unicode Standard. These classes are also printed in Chapter 4 of the Unicode Standard.\",\n",
    "    ),\n",
    "    (\n",
    "        \"Bidirectional category\",\n",
    "        \"normative\",\n",
    "        \"See the list below for an explanation of the abbreviations used in this field. These are the categories required by the Bidirectional Behavior Algorithm in the Unicode Standard. These categories are summarized in Chapter 3 of the Unicode Standard.\",\n",
    "    ),\n",
    "    (\n",
    "        \"Character decomposition mapping\",\n",
    "        \"normative\",\n",
    "        \"In the Unicode Standard, not all of the mappings are full (maximal) decompositions. Recursive application of look-up for decompositions will, in all cases, lead to a maximal decomposition. The decomposition mappings match exactly the decomposition mappings published with the character names in the Unicode Standard.\",\n",
    "    ),\n",
    "    (\n",
    "        \"Decimal digit value\",\n",
    "        \"normative\",\n",
    "        \"This is a numeric field. If the character has the decimal digit property, as specified in Chapter 4 of the Unicode Standard, the value of that digit is represented with an integer value in this field\",\n",
    "    ),\n",
    "    (\n",
    "        \"Digit value\",\n",
    "        \"normative\",\n",
    "        \"This is a numeric field. If the character represents a digit, not necessarily a decimal digit, the value is here. This covers digits which do not form decimal radix forms, such as the compatibility superscript digits\",\n",
    "    ),\n",
    "    (\n",
    "        \"Numeric value\",\n",
    "        \"normative\",\n",
    "        'This is a numeric field. If the character has the numeric property, as specified in Chapter 4 of the Unicode Standard, the value of that character is represented with an integer or rational number in this field. This includes fractions as, e.g., \"1/5\" for U+2155 VULGAR FRACTION ONE FIFTH Also included are numerical values for compatibility characters such as circled numbers.',\n",
    "    ),\n",
    "    (\n",
    "        \"Mirrored\",\n",
    "        \"normative\",\n",
    "        'If the character has been identified as a \"mirrored\" character in bidirectional text, this field has the value \"Y\"; otherwise \"N\". The list of mirrored characters is also printed in Chapter 4 of the Unicode Standard.',\n",
    "    ),\n",
    "    (\n",
    "        \"Unicode 1.0 Name\",\n",
    "        \"informative\",\n",
    "        \"This is the old name as published in Unicode 1.0. This name is only provided when it is significantly different from the Unicode 3.0 name for the character.\",\n",
    "    ),\n",
    "    (\n",
    "        \"10646 comment field\",\n",
    "        \"informative\",\n",
    "        \"This is the ISO 10646 comment field. It is in parantheses in the 10646 names list.\",\n",
    "    ),\n",
    "    (\n",
    "        \"Uppercase mapping\",\n",
    "        \"informative\",\n",
    "        \"Upper case equivalent mapping. If a character is part of an alphabet with case distinctions, and has an upper case equivalent, then the upper case equivalent is in this field. See the explanation below on case distinctions. These mappings are always one-to-one, not one-to-many or many-to-one. This field is informative.\",\n",
    "    ),\n",
    "    (\"Lowercase mapping\", \"informative\", \"Similar to Uppercase mapping\"),\n",
    "    (\n",
    "        \"Titlecase mapping\",\n",
    "        \"informative\",\n",
    "        'Similar to Uppercase mapping\"Titlecase mapping\", \"informative\", \"Similar to Uppercase mapping\"',\n",
    "    ),\n",
    "]\n",
    "\n",
    "metadata = pd.DataFrame(raw_metadata, columns=[\"Field Name\", \"Status\", \"Explanation\"])\n",
    "metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be1c40cb",
   "metadata": {},
   "source": [
    "## Read data file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0eebc389-d590-4843-b247-20cda0f4f152",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "UCD = pd.read_csv(UCD_PATH / \"unicodedata.txt\", sep=\";\", names=metadata[\"Field Name\"])\n",
    "UCD = UCD.assign(code_point=UCD[\"Code value\"].map(hex2int)).set_index(\"code_point\")\n",
    "UCD"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64ecf9df",
   "metadata": {},
   "source": [
    "## Read blocks.txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f98ed8e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "blocks = pd.read_csv(\n",
    "    UCD_PATH / \"blocks.txt\",\n",
    "    sep=\";\",\n",
    "    comment=\"#\",\n",
    "    names=[\"block\", \"block_name\"],\n",
    ")\n",
    "\n",
    "blocks = blocks.assign(\n",
    "    block_name=blocks.block_name.str.strip(),\n",
    "    **blocks.block.str.split(\"..\", regex=False, expand=True)\n",
    "    .rename(columns={0: \"start\", 1: \"end\"})\n",
    "    .map(hex2int),\n",
    ")\n",
    "blocks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c627117b",
   "metadata": {},
   "source": [
    "## Preprocess the blocks data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8fba3d3-263c-471c-bab5-651f07e02903",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class UnicodeEntry(NamedTuple):\n",
    "    code_point: str\n",
    "    symbol: str\n",
    "    block: str\n",
    "    block_name: str\n",
    "    name: str\n",
    "    tab_completion: Optional[str] = None\n",
    "    latex_code: Optional[str] = None\n",
    "\n",
    "\n",
    "def format_block(s: str, /, digits=6) -> str:\n",
    "    lower, upper, *rest = s.split(\"..\")\n",
    "    assert not rest\n",
    "    lower = hex_format(hex2int(lower), digits=6)\n",
    "    upper = hex_format(hex2int(upper), digits=6)\n",
    "    return f\"[{'-'.join([lower, upper])}]\"\n",
    "\n",
    "\n",
    "def format_name(s: str, /) -> str:\n",
    "    return s.lower().replace(\" \", \"_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "234b5e41-5a69-4e2d-9865-54b4f8652f23",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "processed_blocks = {}\n",
    "for _, (block, block_name, start, end) in blocks.iterrows():\n",
    "    # gather all entries in the current block\n",
    "    entries = [\n",
    "        UnicodeEntry(\n",
    "            code_point=f\"U+{hex_format(k)}\",\n",
    "            symbol=f\"{chr(k)}\",\n",
    "            block=block,\n",
    "            block_name=block_name,\n",
    "            name=str(UCD.loc[k, \"Character name\"]).capitalize(),\n",
    "        )\n",
    "        for k in range(start, end + 1)\n",
    "        if k in UCD.index\n",
    "    ]\n",
    "    block_range = format_block(block)\n",
    "    block_name = format_name(block_name)\n",
    "    processed_blocks[(block_range, block_name)] = pd.DataFrame(entries)\n",
    "\n",
    "next(iter(processed_blocks.items()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79f5799d",
   "metadata": {},
   "source": [
    "## Export the blocks to tsv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8ad9ff9-b5c3-4d6b-8b2e-f5caff6a00ad",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ICON_PATH.mkdir(exist_ok=True, parents=True)\n",
    "for (block, name), payload in processed_blocks.items():\n",
    "    fname = f\"{block}-{name}.tsv\"\n",
    "    try:\n",
    "        payload.to_csv(\n",
    "            ICON_PATH / fname,\n",
    "            index=False,\n",
    "            sep=\"\\t\",\n",
    "        )\n",
    "    except Exception as exc:\n",
    "        warnings.warn(f\"❌ Failed to save {fname} due to {exc}\")\n",
    "        continue\n",
    "    else:\n",
    "        print(f\"✅ Saved {fname}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59280b35",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}