OpenCVとTypeScriptの組み合わせで快適に開発するためにJupyter Notebookを使う

$ npm install -g tslab
....
$ tslab install
Running python3 /usr/local/share/npm-global/lib/node_modules/tslab/python/install.py --tslab=tslab
Installing TypeScript kernel spec
Installing JavaScript kernel spec
$ tslab --version
tslab 1.0.21

$ jupyter kernelspec list
Available kernels:
  jslab      /home/node/.local/share/jupyter/kernels/jslab
  tslab      /home/node/.local/share/jupyter/kernels/tslab
  python3    /usr/share/jupyter/kernels/python3

$ jupyter notebook
[I 01:46:45.914 NotebookApp] Serving notebooks from local directory: /workspaces/sample-node-typescript-opencv
[I 01:46:45.914 NotebookApp] Jupyter Notebook 6.2.0 is running at:
[I 01:46:45.914 NotebookApp] http://localhost:8888/?token=7f61e3425b775165bd12ac3e9f9e5908b038f2402f5c026e
[I 01:46:45.914 NotebookApp]  or http://127.0.0.1:8888/?token=7f61e3425b775165bd12ac3e9f9e5908b038f2402f5c026e
[I 01:46:45.914 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).
[C 01:46:45.921 NotebookApp] 
    
    To access the notebook, open this file in a browser:
        file:///home/node/.local/share/jupyter/runtime/nbserver-10364-open.html
    Or copy and paste one of these URLs:
        http://localhost:8888/?token=7f61e3425b775165bd12ac3e9f9e5908b038f2402f5c026e
     or http://127.0.0.1:8888/?token=7f61e3425b775165bd12ac3e9f9e5908b038f2402f5c026e

npm i @techstark/opencv-js jimp tslab

import { display } from "tslab"
import Jimp from "jimp"
import cv from "@techstark/opencv-js"

function toImage(imageMat: cv.Mat): Jimp {
  return new Jimp({
    width: imageMat.cols,
    height: imageMat.rows,
    data: Buffer.from(imageMat.data)
  })
}
async function print(imageMat: cv.Mat) {
  const invoke = async () => {
    const outputImage = toImage(imageMat)
    display.png((await outputImage.getBufferAsync(outputImage.getMIME())))
  }
  if (imageMat.type() === cv.CV_8UC1) {
    cv.cvtColor(imageMat, imageMat, cv.COLOR_GRAY2RGBA, 0)
    await invoke()
    cv.cvtColor(imageMat, imageMat, cv.COLOR_RGBA2GRAY, 0)
    return
  }
  await invoke()
}

const image = await Jimp.read("../images/apple.jpg")
const imageMat = cv.matFromImageData(image.bitmap)
await print(imageMat)

const workImage = new cv.Mat()
cv.cvtColor(imageMat, workImage, cv.COLOR_RGBA2GRAY, 0)
await print(workImage)

npm i tesseract.js

 import { display } from "tslab"
 import Jimp from "jimp"
 import cv from "@techstark/opencv-js"
+import { createWorker } from "tesseract.js"

-const image = await Jimp.read("../images/apple.jpg")
+const image = await Jimp.read("../images/udonjouhou_data1-1.png")
 const imageMat = cv.matFromImageData(image.bitmap)
 await print(imageMat)
-
-const workImage = new cv.Mat()
-cv.cvtColor(imageMat, workImage, cv.COLOR_RGBA2GRAY, 0)
-await print(workImage)

const workImage = imageMat.clone()
// grayscale
cv.cvtColor(workImage, workImage, cv.COLOR_RGBA2GRAY, 0)
await print(workImage)

// threshold
const thresh = 220
cv.threshold(
  workImage,
  workImage,
  thresh,
  255,
  cv.THRESH_BINARY
)
await print(workImage)

// Contour detection
const contours = new cv.MatVector()
const hierarchy = new cv.Mat()
cv.findContours(
  workImage,
  contours,
  hierarchy,
  cv.RETR_TREE,
  cv.CHAIN_APPROX_SIMPLE
)

const overrideImageMat = imageMat.clone()
const rects: cv.Rect[] = []
for (let i = 0; i < contours.size(); i++) {
  // 輪郭の面積をもとに、セルらしい面積のものだけを処理する
  // 実務では、もう少し条件を考えたほうが良い
  const area = cv.contourArea(contours.get(i))
  if (area < 1000 || area > 35000) {
    continue
  }

  const rect = cv.boundingRect(contours.get(i))
  rects.push(rect)

  const color = new cv.Scalar(
    Math.random() * 255,
    Math.random() * 255,
    Math.random() * 255,
    255
  )
  cv.drawContours(overrideImageMat, contours, i, color, 3)
}

await print(overrideImageMat)

const worker = await createWorker("jpn")
try {
  const ocrData: { text: string, image: cv.Mat }[] = []
  for (const rect of rects) {
    const cellImageMat = imageMat.roi(rect).clone()
    const cellImage = toImage(cellImageMat)
    const result = await worker.recognize(await cellImage.getBufferAsync(cellImage.getMIME()))

    ocrData.push({
      text: result.data.text,
      image: cellImageMat
    })
  }

  for (let i = 0; i < 5; i++) {
    const ocrDatum = ocrData[i]
    await print(ocrDatum.image)
    console.log(ocrDatum.text)
  }
} finally {
  worker.terminate()
}