Učitavanje velikih datoteka

FILE=training-video-transcripts.zip
SIZE=$(wc -c < "$FILE" | tr -d ' ')

curl -X POST https://api.sursa.com/v1/uploads \
  -H "Authorization: Bearer $SURSA_API_KEY" \
  -H "Idempotency-Key: $(uuidgen)" \
  -H "Content-Type: application/json" \
  -d "{\"filename\": \"$FILE\", \"content_type\": \"application/zip\", \"size\": $SIZE, \"purpose\": \"archive\"}"

UPLOAD=upl_5d3c2b1a0f9e8d7c6b5a493827161504
split -b 8388608 -a 4 "$FILE" part-

n=1
for part in part-*; do
  curl -X PUT "https://api.sursa.com/v1/uploads/$UPLOAD/parts/$n" \
    -H "Authorization: Bearer $SURSA_API_KEY" \
    -H "Content-Type: application/octet-stream" \
    --data-binary "@$part" --retry 5 --fail
  n=$((n + 1))
done

SHA=$(shasum -a 256 "$FILE" | cut -d' ' -f1)
curl -X POST "https://api.sursa.com/v1/uploads/$UPLOAD/complete" \
  -H "Authorization: Bearer $SURSA_API_KEY" \
  -H "Idempotency-Key: complete-$UPLOAD" \
  -H "Content-Type: application/json" \
  -d "{\"sha256\": \"$SHA\"}"

# A card from the uploaded file
curl -X POST https://api.sursa.com/v1/cards \
  -H "Authorization: Bearer $SURSA_API_KEY" \
  -H "Idempotency-Key: card-from-$UPLOAD" \
  -H "Content-Type: application/json" \
  -d "{\"upload_id\": \"$UPLOAD\", \"collection\": \"support\", \"async\": true}"

# An archive of documents from the uploaded ZIP
curl -X POST https://api.sursa.com/v1/cards/archive \
  -H "Authorization: Bearer $SURSA_API_KEY" \
  -H "Idempotency-Key: archive-from-$UPLOAD" \
  -H "Content-Type: application/json" \
  -d "{\"upload_id\": \"$UPLOAD\", \"collection\": \"support\"}"

import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { open, stat } from "node:fs/promises";
import { basename } from "node:path";

const BASE = "https://api.sursa.com";
const auth = { Authorization: `Bearer ${process.env.SURSA_API_KEY}` };

async function call(method, path, body, headers = {}) {
  const isJson = body !== undefined && !(body instanceof Uint8Array);
  const response = await fetch(BASE + path, {
    method,
    headers: { ...auth, ...(isJson ? { "Content-Type": "application/json" } : {}), ...headers },
    body: isJson ? JSON.stringify(body) : body,
  });
  if (!response.ok) throw new Error(`${method} ${path}: HTTP ${response.status} ${await response.text()}`);
  return response.json();
}

export async function uploadLargeFile(file, contentType, purpose, parallel = 4) {
  const { size } = await stat(file);
  const upload = await call("POST", "/v1/uploads",
    { filename: basename(file), content_type: contentType, size, purpose },
    { "Idempotency-Key": crypto.randomUUID() });

  const hash = createHash("sha256");
  for await (const block of createReadStream(file)) hash.update(block);

  const handle = await open(file);

  let next = 1;
  async function worker() {
    for (let n = next++; n <= upload.part_count; n = next++) {
      const buffer = new Uint8Array(Math.min(upload.part_size, size - (n - 1) * upload.part_size));
      await handle.read(buffer, 0, buffer.length, (n - 1) * upload.part_size);
      for (let attempt = 1; ; attempt++) {
        try {
          await call("PUT", `/v1/uploads/${upload.id}/parts/${n}`, buffer, { "Content-Type": "application/octet-stream" });
          break;
        } catch (error) {
          if (attempt === 5) throw error;
          await new Promise((resolve) => setTimeout(resolve, 1000 * 2 ** attempt));
        }
      }
    }
  }
  await Promise.all(Array.from({ length: parallel }, worker));
  await handle.close();

  await call("POST", `/v1/uploads/${upload.id}/complete`, { sha256: hash.digest("hex") },
    { "Idempotency-Key": `complete-${upload.id}` });
  for (;;) {
    const state = await call("GET", `/v1/uploads/${upload.id}`);
    if (state.status === "ready") return upload.id;
    if (state.status === "rejected") throw new Error(`upload rejected: ${state.reject_reason}`);
    await new Promise((resolve) => setTimeout(resolve, 2000));
  }
}