Task #526: Off-Replit migration & GitHub-ready cleanup
Fully decoupled Tx OS from the Replit hosted environment so the project can be cloned and run on any Linux VPS with `docker compose up`. Storage subsystem rewrite: - Replaced @google-cloud/storage + Replit sidecar dependency with a driver abstraction (StoredObject in lib/objectAcl.ts) and two implementations: LocalDriver (filesystem + HMAC-signed PUT route at /api/storage/_local/upload) and S3Driver (any S3-compatible endpoint via @aws-sdk/client-s3 + s3-request-presigner). Driver auto-selected by STORAGE_DRIVER / S3_ENDPOINT env vars. - Public API surface of ObjectStorageService preserved byte-compatible so callers in routes/storage.ts and routes/executive-meetings.ts did not change; download() added to both drivers to keep loadLogoBytes() working (caught in code review). - Storage object-authz tests A-L (incl. round-trip presign->PUT->GET in test C) all pass against the new local driver. Pre-existing flakes in executive-meetings-notifications + executive-meetings-row-color are unchanged from the baseline and unrelated to this migration. Infrastructure: - Dockerfile (5 targets: deps/build/api/web/migrate). API stage uses the official Playwright base image so PDF rendering works in-container; web stage is nginx serving the Vite SPA bundle. - docker-compose.yml: postgres + minio + minio-init (creates buckets) + api + web + one-shot migrate runner. Healthchecks on every long-lived service. - docker/nginx.conf: SPA fallback, /api proxy, /api/socket.io websocket upgrade ordering. - .env.example: every runtime env var documented with comments. - README.md replaces replit.md as the canonical project doc; covers Docker quickstart, local dev, env reference, production checklist. - MIGRATION_REPORT.md: file-by-file diff of what changed and why. Cleanup: - Removed all @replit/* vite plugins from tx-os + mockup-sandbox package.json + vite.config.ts + pnpm-workspace.yaml catalog. - Removed @google-cloud/storage and google-auth-library from api-server. - Deleted attached_assets/ (23MB), sedMkjeJm temp file, stale dist/ and *.tsbuildinfo build artefacts, scripts/post-merge.sh. - Stripped Replit references from threat_model.md (now describes the self-hosted topology). - New comprehensive .gitignore: Replit configs (.replit, replit.nix, replit.md), agent state (.local/, .canvas/, .agents/, .cache/, .config/, .upm/), local storage/, .env*, build artefacts. .replit and replit.nix remain on disk (sandbox-protected) but will not ship to GitHub. - scripts/src/seed.ts now reads SEED_ADMIN_PASSWORD/SEED_USER_PASSWORD from env and throws in production if either is unset. Drift from plan: replit.md was deleted (per off-Replit scope) so its "do-not-touch files" preference list is moot. .replit/replit.nix kept on disk only because they are sandbox-protected from edit/delete in this environment, but they are git-ignored so they will not appear in a fresh clone.
This commit is contained in:
riyadhafraa
@@ -12,7 +12,8 @@
|
||||
"test": "node --test 'tests/**/*.test.mjs'"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google-cloud/storage": "^7.19.0",
|
||||
"@aws-sdk/client-s3": "^3.658.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.658.0",
|
||||
"@swc/helpers": "^0.5.21",
|
||||
"@workspace/api-zod": "workspace:*",
|
||||
"@workspace/db": "workspace:*",
|
||||
@@ -26,7 +27,6 @@
|
||||
"express": "^5",
|
||||
"express-rate-limit": "^8.5.1",
|
||||
"express-session": "^1.19.0",
|
||||
"google-auth-library": "^10.6.2",
|
||||
"nodemailer": "^8.0.7",
|
||||
"pdfkit": "^0.18.0",
|
||||
"pino": "^9",
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import { File } from "@google-cloud/storage";
|
||||
|
||||
// =====================================================================
|
||||
// DEPRECATED — DO NOT USE FOR ACCESS DECISIONS
|
||||
// =====================================================================
|
||||
@@ -20,14 +18,12 @@ import { File } from "@google-cloud/storage";
|
||||
// What remains here is the bare minimum needed to keep
|
||||
// `objectStorage.downloadObject` working: it consults the cached ACL
|
||||
// metadata only to decide between `Cache-Control: public` and
|
||||
// `Cache-Control: private` headers. Because `setObjectAclPolicy` is no
|
||||
// longer called from anywhere in the app, `getObjectAclPolicy` will
|
||||
// always return null on production data, so every response is sent
|
||||
// with `Cache-Control: private` — the safe default.
|
||||
// `Cache-Control: private` headers. Because `setAclPolicy` is no
|
||||
// longer called from anywhere in the app, `getAclPolicy` will always
|
||||
// return null on production data, so every response is sent with
|
||||
// `Cache-Control: private` — the safe default.
|
||||
// =====================================================================
|
||||
|
||||
const ACL_POLICY_METADATA_KEY = "custom:aclPolicy";
|
||||
|
||||
export enum ObjectPermission {
|
||||
READ = "read",
|
||||
WRITE = "write",
|
||||
@@ -41,34 +37,29 @@ export interface ObjectAclPolicy {
|
||||
visibility: "public" | "private";
|
||||
}
|
||||
|
||||
export async function setObjectAclPolicy(
|
||||
objectFile: File,
|
||||
aclPolicy: ObjectAclPolicy,
|
||||
): Promise<void> {
|
||||
const [exists] = await objectFile.exists();
|
||||
if (!exists) {
|
||||
throw new Error(`Object not found: ${objectFile.name}`);
|
||||
}
|
||||
await objectFile.setMetadata({
|
||||
metadata: {
|
||||
[ACL_POLICY_METADATA_KEY]: JSON.stringify(aclPolicy),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function getObjectAclPolicy(
|
||||
objectFile: File,
|
||||
): Promise<ObjectAclPolicy | null> {
|
||||
const [metadata] = await objectFile.getMetadata();
|
||||
const aclPolicy = metadata?.metadata?.[ACL_POLICY_METADATA_KEY];
|
||||
if (!aclPolicy) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(aclPolicy as string) as ObjectAclPolicy;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Driver-agnostic stored-object handle. Replaces the old `File` type
|
||||
* from `@google-cloud/storage` so route handlers (which operate on
|
||||
* the result opaquely) don't depend on a specific storage backend.
|
||||
*/
|
||||
export interface StoredObject {
|
||||
bucketName: string;
|
||||
objectName: string;
|
||||
exists(): Promise<boolean>;
|
||||
getMetadata(): Promise<{
|
||||
contentType?: string;
|
||||
size?: number;
|
||||
aclPolicy?: ObjectAclPolicy | null;
|
||||
}>;
|
||||
createReadStream(): NodeJS.ReadableStream;
|
||||
/**
|
||||
* Reads the entire object into a Buffer. The single-element tuple shape
|
||||
* mirrors `@google-cloud/storage`'s `File.download()` so legacy callers
|
||||
* (e.g. PDF logo loader) keep working without refactoring.
|
||||
*/
|
||||
download(): Promise<[Buffer]>;
|
||||
setAclPolicy(policy: ObjectAclPolicy): Promise<void>;
|
||||
getAclPolicy(): Promise<ObjectAclPolicy | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -78,7 +69,7 @@ export async function getObjectAclPolicy(
|
||||
*/
|
||||
export async function canAccessObject(_args: {
|
||||
userId?: string;
|
||||
objectFile: File;
|
||||
objectFile: StoredObject;
|
||||
requestedPermission: ObjectPermission;
|
||||
}): Promise<boolean> {
|
||||
return false;
|
||||
|
||||
@@ -1,34 +1,22 @@
|
||||
import { Storage, File } from "@google-cloud/storage";
|
||||
import { Readable } from "stream";
|
||||
import { randomUUID } from "crypto";
|
||||
import { randomUUID, createHmac, timingSafeEqual } from "crypto";
|
||||
import { promises as fs, createReadStream, createWriteStream } from "fs";
|
||||
import path from "path";
|
||||
import {
|
||||
S3Client,
|
||||
HeadObjectCommand,
|
||||
GetObjectCommand,
|
||||
PutObjectCommand,
|
||||
CopyObjectCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
import {
|
||||
ObjectAclPolicy,
|
||||
ObjectPermission,
|
||||
canAccessObject,
|
||||
getObjectAclPolicy,
|
||||
setObjectAclPolicy,
|
||||
type StoredObject,
|
||||
} from "./objectAcl";
|
||||
|
||||
const REPLIT_SIDECAR_ENDPOINT = "http://127.0.0.1:1106";
|
||||
|
||||
export const objectStorageClient = new Storage({
|
||||
credentials: {
|
||||
audience: "replit",
|
||||
subject_token_type: "access_token",
|
||||
token_url: `${REPLIT_SIDECAR_ENDPOINT}/token`,
|
||||
type: "external_account",
|
||||
credential_source: {
|
||||
url: `${REPLIT_SIDECAR_ENDPOINT}/credential`,
|
||||
format: {
|
||||
type: "json",
|
||||
subject_token_field_name: "access_token",
|
||||
},
|
||||
},
|
||||
universe_domain: "googleapis.com",
|
||||
},
|
||||
projectId: "",
|
||||
});
|
||||
|
||||
export class ObjectNotFoundError extends Error {
|
||||
constructor() {
|
||||
super("Object not found");
|
||||
@@ -37,6 +25,391 @@ export class ObjectNotFoundError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Driver interface
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface StorageDriver {
|
||||
getObject(bucketName: string, objectName: string): StoredObject;
|
||||
signUploadUrl(bucketName: string, objectName: string, ttlSec: number): Promise<string>;
|
||||
/** Recognise a previously-issued upload URL and return the object path. */
|
||||
parseUploadUrl(rawUrl: string): { bucketName: string; objectName: string } | null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Local-FS driver — used when no S3_ENDPOINT is configured. Backs storage on
|
||||
// the host filesystem under ./storage/<bucket>/<object>. Signed upload URLs
|
||||
// are HMAC tokens served by the local API at /api/storage/_local/upload.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const LOCAL_STORAGE_ROOT = path.resolve(
|
||||
process.env.LOCAL_STORAGE_ROOT ?? "./storage",
|
||||
);
|
||||
const LOCAL_HMAC_SECRET = (() => {
|
||||
const s = process.env.LOCAL_STORAGE_SIGNING_SECRET ?? process.env.SESSION_SECRET;
|
||||
if (s) return s;
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
throw new Error(
|
||||
"LOCAL_STORAGE_SIGNING_SECRET (or SESSION_SECRET) must be set in production " +
|
||||
"when using the local storage driver.",
|
||||
);
|
||||
}
|
||||
return "tx-dev-local-storage-secret-change-before-deploy";
|
||||
})();
|
||||
|
||||
function localFsPath(bucketName: string, objectName: string): string {
|
||||
// Reject path traversal — joining and re-checking the absolute path
|
||||
// must remain inside LOCAL_STORAGE_ROOT.
|
||||
const joined = path.resolve(LOCAL_STORAGE_ROOT, bucketName, objectName);
|
||||
if (!joined.startsWith(LOCAL_STORAGE_ROOT + path.sep) && joined !== LOCAL_STORAGE_ROOT) {
|
||||
throw new Error("Invalid object path");
|
||||
}
|
||||
return joined;
|
||||
}
|
||||
|
||||
function localMetadataPath(bucketName: string, objectName: string): string {
|
||||
return localFsPath(bucketName, objectName) + ".meta.json";
|
||||
}
|
||||
|
||||
interface LocalMetadataFile {
|
||||
contentType?: string;
|
||||
aclPolicy?: ObjectAclPolicy | null;
|
||||
}
|
||||
|
||||
class LocalStoredObject implements StoredObject {
|
||||
constructor(public bucketName: string, public objectName: string) {}
|
||||
|
||||
async exists(): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(localFsPath(this.bucketName, this.objectName));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(): Promise<{
|
||||
contentType?: string;
|
||||
size?: number;
|
||||
aclPolicy?: ObjectAclPolicy | null;
|
||||
}> {
|
||||
const fp = localFsPath(this.bucketName, this.objectName);
|
||||
const stat = await fs.stat(fp);
|
||||
let meta: LocalMetadataFile = {};
|
||||
try {
|
||||
const raw = await fs.readFile(localMetadataPath(this.bucketName, this.objectName), "utf8");
|
||||
meta = JSON.parse(raw) as LocalMetadataFile;
|
||||
} catch {
|
||||
// No sidecar metadata file — that's fine.
|
||||
}
|
||||
return {
|
||||
contentType: meta.contentType,
|
||||
size: stat.size,
|
||||
aclPolicy: meta.aclPolicy ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
createReadStream(): NodeJS.ReadableStream {
|
||||
return createReadStream(localFsPath(this.bucketName, this.objectName));
|
||||
}
|
||||
|
||||
async download(): Promise<[Buffer]> {
|
||||
const buf = await fs.readFile(localFsPath(this.bucketName, this.objectName));
|
||||
return [buf];
|
||||
}
|
||||
|
||||
async setAclPolicy(policy: ObjectAclPolicy): Promise<void> {
|
||||
if (!(await this.exists())) {
|
||||
throw new Error(`Object not found: ${this.objectName}`);
|
||||
}
|
||||
let meta: LocalMetadataFile = {};
|
||||
try {
|
||||
meta = JSON.parse(
|
||||
await fs.readFile(localMetadataPath(this.bucketName, this.objectName), "utf8"),
|
||||
) as LocalMetadataFile;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
meta.aclPolicy = policy;
|
||||
await fs.writeFile(
|
||||
localMetadataPath(this.bucketName, this.objectName),
|
||||
JSON.stringify(meta),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
|
||||
async getAclPolicy(): Promise<ObjectAclPolicy | null> {
|
||||
const m = await this.getMetadata();
|
||||
return m.aclPolicy ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
class LocalDriver implements StorageDriver {
|
||||
getObject(bucketName: string, objectName: string): StoredObject {
|
||||
return new LocalStoredObject(bucketName, objectName);
|
||||
}
|
||||
|
||||
async signUploadUrl(
|
||||
bucketName: string,
|
||||
objectName: string,
|
||||
ttlSec: number,
|
||||
): Promise<string> {
|
||||
const exp = Math.floor(Date.now() / 1000) + ttlSec;
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({ b: bucketName, o: objectName, exp }),
|
||||
).toString("base64url");
|
||||
const sig = createHmac("sha256", LOCAL_HMAC_SECRET).update(payload).digest("base64url");
|
||||
const token = `${payload}.${sig}`;
|
||||
const base = (process.env.PUBLIC_BASE_URL ?? `http://localhost:${process.env.PORT ?? "8080"}`).replace(/\/+$/, "");
|
||||
return `${base}/api/storage/_local/upload?token=${token}`;
|
||||
}
|
||||
|
||||
parseUploadUrl(rawUrl: string): { bucketName: string; objectName: string } | null {
|
||||
try {
|
||||
const u = new URL(rawUrl);
|
||||
if (!u.pathname.endsWith("/api/storage/_local/upload")) return null;
|
||||
const token = u.searchParams.get("token");
|
||||
if (!token) return null;
|
||||
const verified = verifyLocalUploadToken(token);
|
||||
if (!verified) return null;
|
||||
return { bucketName: verified.b, objectName: verified.o };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a local-driver upload token. Returns the decoded payload on
|
||||
* success (signature match + not expired), null otherwise.
|
||||
*/
|
||||
export function verifyLocalUploadToken(
|
||||
token: string,
|
||||
): { b: string; o: string; exp: number } | null {
|
||||
const parts = token.split(".");
|
||||
if (parts.length !== 2) return null;
|
||||
const [payload, sig] = parts;
|
||||
const expectedSig = createHmac("sha256", LOCAL_HMAC_SECRET).update(payload).digest("base64url");
|
||||
const a = Buffer.from(sig);
|
||||
const b = Buffer.from(expectedSig);
|
||||
if (a.length !== b.length || !timingSafeEqual(a, b)) return null;
|
||||
let decoded: { b: string; o: string; exp: number };
|
||||
try {
|
||||
decoded = JSON.parse(Buffer.from(payload, "base64url").toString("utf8")) as {
|
||||
b: string;
|
||||
o: string;
|
||||
exp: number;
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof decoded.exp !== "number" || decoded.exp < Math.floor(Date.now() / 1000)) {
|
||||
return null;
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist an upload to the local-FS storage backend. Used by the
|
||||
* /api/storage/_local/upload route.
|
||||
*/
|
||||
export async function writeLocalObject(
|
||||
bucketName: string,
|
||||
objectName: string,
|
||||
body: Readable,
|
||||
contentType?: string,
|
||||
): Promise<void> {
|
||||
const fp = localFsPath(bucketName, objectName);
|
||||
await fs.mkdir(path.dirname(fp), { recursive: true });
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ws = createWriteStream(fp);
|
||||
body.pipe(ws);
|
||||
ws.on("finish", () => resolve());
|
||||
ws.on("error", reject);
|
||||
body.on("error", reject);
|
||||
});
|
||||
if (contentType) {
|
||||
let meta: LocalMetadataFile = {};
|
||||
try {
|
||||
meta = JSON.parse(await fs.readFile(localMetadataPath(bucketName, objectName), "utf8")) as LocalMetadataFile;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
meta.contentType = contentType;
|
||||
await fs.writeFile(localMetadataPath(bucketName, objectName), JSON.stringify(meta), "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// S3 driver — used when S3_ENDPOINT is configured. Targets MinIO or any
|
||||
// S3-compatible bucket via @aws-sdk/client-s3 + s3-request-presigner.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ACL_METADATA_HEADER = "x-amz-meta-acl-policy";
|
||||
|
||||
let s3ClientSingleton: S3Client | null = null;
|
||||
function getS3Client(): S3Client {
|
||||
if (s3ClientSingleton) return s3ClientSingleton;
|
||||
const endpoint = process.env.S3_ENDPOINT;
|
||||
if (!endpoint) throw new Error("S3_ENDPOINT not set");
|
||||
const region = process.env.S3_REGION ?? "us-east-1";
|
||||
const accessKeyId = process.env.S3_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY;
|
||||
if (!accessKeyId || !secretAccessKey) {
|
||||
throw new Error("S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY must be set when S3_ENDPOINT is configured");
|
||||
}
|
||||
s3ClientSingleton = new S3Client({
|
||||
endpoint,
|
||||
region,
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
forcePathStyle: process.env.S3_FORCE_PATH_STYLE !== "false",
|
||||
});
|
||||
return s3ClientSingleton;
|
||||
}
|
||||
|
||||
class S3StoredObject implements StoredObject {
|
||||
constructor(public bucketName: string, public objectName: string) {}
|
||||
|
||||
async exists(): Promise<boolean> {
|
||||
try {
|
||||
await getS3Client().send(
|
||||
new HeadObjectCommand({ Bucket: this.bucketName, Key: this.objectName }),
|
||||
);
|
||||
return true;
|
||||
} catch (err) {
|
||||
const name = (err as Error & { name?: string }).name;
|
||||
if (name === "NotFound" || name === "NoSuchKey") return false;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(): Promise<{
|
||||
contentType?: string;
|
||||
size?: number;
|
||||
aclPolicy?: ObjectAclPolicy | null;
|
||||
}> {
|
||||
const head = await getS3Client().send(
|
||||
new HeadObjectCommand({ Bucket: this.bucketName, Key: this.objectName }),
|
||||
);
|
||||
const aclRaw = head.Metadata?.["acl-policy"];
|
||||
let aclPolicy: ObjectAclPolicy | null = null;
|
||||
if (aclRaw) {
|
||||
try {
|
||||
aclPolicy = JSON.parse(aclRaw) as ObjectAclPolicy;
|
||||
} catch {
|
||||
aclPolicy = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
contentType: head.ContentType,
|
||||
size: head.ContentLength,
|
||||
aclPolicy,
|
||||
};
|
||||
}
|
||||
|
||||
createReadStream(): NodeJS.ReadableStream {
|
||||
// Lazy stream: synchronously return a PassThrough that we'll pipe
|
||||
// into once the GetObject promise resolves.
|
||||
const { PassThrough } = require("stream") as typeof import("stream");
|
||||
const out = new PassThrough();
|
||||
getS3Client()
|
||||
.send(new GetObjectCommand({ Bucket: this.bucketName, Key: this.objectName }))
|
||||
.then((res) => {
|
||||
const body = res.Body as Readable | undefined;
|
||||
if (!body) {
|
||||
out.end();
|
||||
return;
|
||||
}
|
||||
body.on("error", (e) => out.destroy(e));
|
||||
body.pipe(out);
|
||||
})
|
||||
.catch((e) => out.destroy(e));
|
||||
return out;
|
||||
}
|
||||
|
||||
async download(): Promise<[Buffer]> {
|
||||
const res = await getS3Client().send(
|
||||
new GetObjectCommand({ Bucket: this.bucketName, Key: this.objectName }),
|
||||
);
|
||||
const body = res.Body as Readable | undefined;
|
||||
if (!body) return [Buffer.alloc(0)];
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of body) {
|
||||
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk));
|
||||
}
|
||||
return [Buffer.concat(chunks)];
|
||||
}
|
||||
|
||||
async setAclPolicy(policy: ObjectAclPolicy): Promise<void> {
|
||||
// S3 supports setting metadata only via copy-self.
|
||||
const json = JSON.stringify(policy);
|
||||
await getS3Client().send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: this.objectName,
|
||||
CopySource: `/${this.bucketName}/${encodeURIComponent(this.objectName)}`,
|
||||
Metadata: { "acl-policy": json },
|
||||
MetadataDirective: "REPLACE",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async getAclPolicy(): Promise<ObjectAclPolicy | null> {
|
||||
return (await this.getMetadata()).aclPolicy ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
class S3Driver implements StorageDriver {
|
||||
getObject(bucketName: string, objectName: string): StoredObject {
|
||||
return new S3StoredObject(bucketName, objectName);
|
||||
}
|
||||
|
||||
async signUploadUrl(
|
||||
bucketName: string,
|
||||
objectName: string,
|
||||
ttlSec: number,
|
||||
): Promise<string> {
|
||||
const cmd = new PutObjectCommand({ Bucket: bucketName, Key: objectName });
|
||||
return getSignedUrl(getS3Client(), cmd, { expiresIn: ttlSec });
|
||||
}
|
||||
|
||||
parseUploadUrl(rawUrl: string): { bucketName: string; objectName: string } | null {
|
||||
try {
|
||||
const u = new URL(rawUrl);
|
||||
// Path-style: /<bucket>/<key...>
|
||||
const segments = u.pathname.replace(/^\/+/, "").split("/");
|
||||
if (segments.length < 2) return null;
|
||||
const bucketName = segments[0];
|
||||
const objectName = segments.slice(1).join("/");
|
||||
return { bucketName, objectName: decodeURIComponent(objectName) };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Driver selection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let driverSingleton: StorageDriver | null = null;
|
||||
function driver(): StorageDriver {
|
||||
if (driverSingleton) return driverSingleton;
|
||||
const explicit = process.env.STORAGE_DRIVER;
|
||||
const useS3 = explicit ? explicit === "s3" : Boolean(process.env.S3_ENDPOINT);
|
||||
driverSingleton = useS3 ? new S3Driver() : new LocalDriver();
|
||||
return driverSingleton;
|
||||
}
|
||||
|
||||
export const ACTIVE_STORAGE_DRIVER = (): "s3" | "local" =>
|
||||
driver() instanceof S3Driver ? "s3" : "local";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ObjectStorageService — public API consumed by routes/storage.ts and
|
||||
// routes/executive-meetings.ts. Surface preserved byte-compatible with the
|
||||
// previous Replit-sidecar implementation.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class ObjectStorageService {
|
||||
constructor() {}
|
||||
|
||||
@@ -46,14 +419,14 @@ export class ObjectStorageService {
|
||||
new Set(
|
||||
pathsStr
|
||||
.split(",")
|
||||
.map((path) => path.trim())
|
||||
.filter((path) => path.length > 0)
|
||||
)
|
||||
.map((p) => p.trim())
|
||||
.filter((p) => p.length > 0),
|
||||
),
|
||||
);
|
||||
if (paths.length === 0) {
|
||||
throw new Error(
|
||||
"PUBLIC_OBJECT_SEARCH_PATHS not set. Create a bucket in 'Object Storage' " +
|
||||
"tool and set PUBLIC_OBJECT_SEARCH_PATHS env var (comma-separated paths)."
|
||||
"PUBLIC_OBJECT_SEARCH_PATHS not set. Set this env var to a comma-separated " +
|
||||
"list of /<bucket>/<prefix> paths (see .env.example).",
|
||||
);
|
||||
}
|
||||
return paths;
|
||||
@@ -63,43 +436,37 @@ export class ObjectStorageService {
|
||||
const dir = process.env.PRIVATE_OBJECT_DIR || "";
|
||||
if (!dir) {
|
||||
throw new Error(
|
||||
"PRIVATE_OBJECT_DIR not set. Create a bucket in 'Object Storage' " +
|
||||
"tool and set PRIVATE_OBJECT_DIR env var."
|
||||
"PRIVATE_OBJECT_DIR not set. Set this env var to /<bucket>/<prefix> " +
|
||||
"(see .env.example).",
|
||||
);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
async searchPublicObject(filePath: string): Promise<File | null> {
|
||||
async searchPublicObject(filePath: string): Promise<StoredObject | null> {
|
||||
for (const searchPath of this.getPublicObjectSearchPaths()) {
|
||||
const fullPath = `${searchPath}/${filePath}`;
|
||||
|
||||
const { bucketName, objectName } = parseObjectPath(fullPath);
|
||||
const bucket = objectStorageClient.bucket(bucketName);
|
||||
const file = bucket.file(objectName);
|
||||
|
||||
const [exists] = await file.exists();
|
||||
if (exists) {
|
||||
return file;
|
||||
const obj = driver().getObject(bucketName, objectName);
|
||||
if (await obj.exists()) {
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async downloadObject(file: File, cacheTtlSec: number = 3600): Promise<Response> {
|
||||
const [metadata] = await file.getMetadata();
|
||||
const aclPolicy = await getObjectAclPolicy(file);
|
||||
const isPublic = aclPolicy?.visibility === "public";
|
||||
async downloadObject(file: StoredObject, cacheTtlSec: number = 3600): Promise<Response> {
|
||||
const metadata = await file.getMetadata();
|
||||
const isPublic = metadata.aclPolicy?.visibility === "public";
|
||||
|
||||
const nodeStream = file.createReadStream();
|
||||
const webStream = Readable.toWeb(nodeStream) as ReadableStream;
|
||||
const webStream = Readable.toWeb(nodeStream as Readable) as ReadableStream;
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": (metadata.contentType as string) || "application/octet-stream",
|
||||
"Content-Type": metadata.contentType ?? "application/octet-stream",
|
||||
"Cache-Control": `${isPublic ? "public" : "private"}, max-age=${cacheTtlSec}`,
|
||||
};
|
||||
if (metadata.size) {
|
||||
if (typeof metadata.size === "number") {
|
||||
headers["Content-Length"] = String(metadata.size);
|
||||
}
|
||||
|
||||
@@ -108,84 +475,68 @@ export class ObjectStorageService {
|
||||
|
||||
async getObjectEntityUploadURL(): Promise<string> {
|
||||
const privateObjectDir = this.getPrivateObjectDir();
|
||||
if (!privateObjectDir) {
|
||||
throw new Error(
|
||||
"PRIVATE_OBJECT_DIR not set. Create a bucket in 'Object Storage' " +
|
||||
"tool and set PRIVATE_OBJECT_DIR env var."
|
||||
);
|
||||
}
|
||||
|
||||
const objectId = randomUUID();
|
||||
const fullPath = `${privateObjectDir}/uploads/${objectId}`;
|
||||
|
||||
const { bucketName, objectName } = parseObjectPath(fullPath);
|
||||
|
||||
return signObjectURL({
|
||||
bucketName,
|
||||
objectName,
|
||||
method: "PUT",
|
||||
ttlSec: 900,
|
||||
});
|
||||
return driver().signUploadUrl(bucketName, objectName, 900);
|
||||
}
|
||||
|
||||
async getObjectEntityFile(objectPath: string): Promise<File> {
|
||||
async getObjectEntityFile(objectPath: string): Promise<StoredObject> {
|
||||
if (!objectPath.startsWith("/objects/")) {
|
||||
throw new ObjectNotFoundError();
|
||||
}
|
||||
|
||||
const parts = objectPath.slice(1).split("/");
|
||||
if (parts.length < 2) {
|
||||
throw new ObjectNotFoundError();
|
||||
}
|
||||
|
||||
const entityId = parts.slice(1).join("/");
|
||||
let entityDir = this.getPrivateObjectDir();
|
||||
if (!entityDir.endsWith("/")) {
|
||||
entityDir = `${entityDir}/`;
|
||||
}
|
||||
if (!entityDir.endsWith("/")) entityDir = `${entityDir}/`;
|
||||
const objectEntityPath = `${entityDir}${entityId}`;
|
||||
const { bucketName, objectName } = parseObjectPath(objectEntityPath);
|
||||
const bucket = objectStorageClient.bucket(bucketName);
|
||||
const objectFile = bucket.file(objectName);
|
||||
const [exists] = await objectFile.exists();
|
||||
if (!exists) {
|
||||
const obj = driver().getObject(bucketName, objectName);
|
||||
if (!(await obj.exists())) {
|
||||
throw new ObjectNotFoundError();
|
||||
}
|
||||
return objectFile;
|
||||
return obj;
|
||||
}
|
||||
|
||||
normalizeObjectEntityPath(rawPath: string): string {
|
||||
if (!rawPath.startsWith("https://storage.googleapis.com/")) {
|
||||
return rawPath;
|
||||
// Try the active driver's URL recogniser first — strip query string,
|
||||
// map a signed-URL back to its /objects/<id> form.
|
||||
const noQuery = rawPath.split("?")[0]!;
|
||||
const parsed = driver().parseUploadUrl(rawPath);
|
||||
let candidatePath = noQuery;
|
||||
if (parsed) {
|
||||
candidatePath = `/${parsed.bucketName}/${parsed.objectName}`;
|
||||
} else {
|
||||
// Legacy GCS URL form (pre-migration data) — strip the host.
|
||||
try {
|
||||
const u = new URL(rawPath);
|
||||
if (u.pathname.startsWith("/")) candidatePath = u.pathname;
|
||||
} catch {
|
||||
// already a relative path
|
||||
}
|
||||
}
|
||||
|
||||
const url = new URL(rawPath);
|
||||
const rawObjectPath = url.pathname;
|
||||
|
||||
let objectEntityDir = this.getPrivateObjectDir();
|
||||
if (!objectEntityDir.endsWith("/")) {
|
||||
objectEntityDir = `${objectEntityDir}/`;
|
||||
}
|
||||
if (!objectEntityDir.endsWith("/")) objectEntityDir = `${objectEntityDir}/`;
|
||||
|
||||
if (!rawObjectPath.startsWith(objectEntityDir)) {
|
||||
return rawObjectPath;
|
||||
if (!candidatePath.startsWith(objectEntityDir)) {
|
||||
return candidatePath;
|
||||
}
|
||||
|
||||
const entityId = rawObjectPath.slice(objectEntityDir.length);
|
||||
const entityId = candidatePath.slice(objectEntityDir.length);
|
||||
return `/objects/${entityId}`;
|
||||
}
|
||||
|
||||
async trySetObjectEntityAclPolicy(
|
||||
rawPath: string,
|
||||
aclPolicy: ObjectAclPolicy
|
||||
aclPolicy: ObjectAclPolicy,
|
||||
): Promise<string> {
|
||||
const normalizedPath = this.normalizeObjectEntityPath(rawPath);
|
||||
if (!normalizedPath.startsWith("/")) {
|
||||
return normalizedPath;
|
||||
}
|
||||
|
||||
if (!normalizedPath.startsWith("/")) return normalizedPath;
|
||||
const objectFile = await this.getObjectEntityFile(normalizedPath);
|
||||
await setObjectAclPolicy(objectFile, aclPolicy);
|
||||
await objectFile.setAclPolicy(aclPolicy);
|
||||
return normalizedPath;
|
||||
}
|
||||
|
||||
@@ -195,7 +546,7 @@ export class ObjectStorageService {
|
||||
requestedPermission,
|
||||
}: {
|
||||
userId?: string;
|
||||
objectFile: File;
|
||||
objectFile: StoredObject;
|
||||
requestedPermission?: ObjectPermission;
|
||||
}): Promise<boolean> {
|
||||
return canAccessObject({
|
||||
@@ -206,62 +557,13 @@ export class ObjectStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
function parseObjectPath(path: string): {
|
||||
bucketName: string;
|
||||
objectName: string;
|
||||
} {
|
||||
if (!path.startsWith("/")) {
|
||||
path = `/${path}`;
|
||||
}
|
||||
const pathParts = path.split("/");
|
||||
function parseObjectPath(p: string): { bucketName: string; objectName: string } {
|
||||
if (!p.startsWith("/")) p = `/${p}`;
|
||||
const pathParts = p.split("/");
|
||||
if (pathParts.length < 3) {
|
||||
throw new Error("Invalid path: must contain at least a bucket name");
|
||||
}
|
||||
|
||||
const bucketName = pathParts[1];
|
||||
const objectName = pathParts.slice(2).join("/");
|
||||
|
||||
return {
|
||||
bucketName,
|
||||
objectName,
|
||||
};
|
||||
}
|
||||
|
||||
async function signObjectURL({
|
||||
bucketName,
|
||||
objectName,
|
||||
method,
|
||||
ttlSec,
|
||||
}: {
|
||||
bucketName: string;
|
||||
objectName: string;
|
||||
method: "GET" | "PUT" | "DELETE" | "HEAD";
|
||||
ttlSec: number;
|
||||
}): Promise<string> {
|
||||
const request = {
|
||||
bucket_name: bucketName,
|
||||
object_name: objectName,
|
||||
method,
|
||||
expires_at: new Date(Date.now() + ttlSec * 1000).toISOString(),
|
||||
};
|
||||
const response = await fetch(
|
||||
`${REPLIT_SIDECAR_ENDPOINT}/object-storage/signed-object-url`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
}
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to sign object URL, errorcode: ${response.status}, ` +
|
||||
`make sure you're running on Replit`
|
||||
);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { signed_url: string };
|
||||
return data.signed_url;
|
||||
return { bucketName, objectName };
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import notificationsRouter from "./notifications";
|
||||
import usersRouter from "./users";
|
||||
import statsRouter from "./stats";
|
||||
import storageRouter from "./storage";
|
||||
import storageLocalUploadRouter from "./storage-local-upload";
|
||||
import settingsRouter from "./settings";
|
||||
import notesRouter from "./notes";
|
||||
import groupsRouter from "./groups";
|
||||
@@ -26,6 +27,7 @@ router.use(notificationsRouter);
|
||||
router.use(usersRouter);
|
||||
router.use(statsRouter);
|
||||
router.use(storageRouter);
|
||||
router.use(storageLocalUploadRouter);
|
||||
router.use(settingsRouter);
|
||||
router.use(notesRouter);
|
||||
router.use(groupsRouter);
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { Router, type IRouter, type Request, type Response } from "express";
|
||||
import { verifyLocalUploadToken, writeLocalObject, ACTIVE_STORAGE_DRIVER } from "../lib/objectStorage";
|
||||
|
||||
const router: IRouter = Router();
|
||||
|
||||
/**
|
||||
* PUT /storage/_local/upload?token=<HMAC>
|
||||
*
|
||||
* Receives the body of a presigned upload issued by the local-FS storage
|
||||
* driver. The token is HMAC-signed with LOCAL_STORAGE_SIGNING_SECRET (or
|
||||
* SESSION_SECRET as fallback) and carries the bucket+object key + expiry.
|
||||
* Only mounted when STORAGE_DRIVER=local (or unset and S3 not configured);
|
||||
* when S3 is the active driver, this endpoint refuses to run.
|
||||
*/
|
||||
router.put("/storage/_local/upload", async (req: Request, res: Response) => {
|
||||
if (ACTIVE_STORAGE_DRIVER() !== "local") {
|
||||
res.status(404).json({ error: "Not found" });
|
||||
return;
|
||||
}
|
||||
const token = typeof req.query.token === "string" ? req.query.token : null;
|
||||
if (!token) {
|
||||
res.status(400).json({ error: "Missing token" });
|
||||
return;
|
||||
}
|
||||
const verified = verifyLocalUploadToken(token);
|
||||
if (!verified) {
|
||||
res.status(403).json({ error: "Invalid or expired upload token" });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const contentType = typeof req.headers["content-type"] === "string"
|
||||
? req.headers["content-type"]
|
||||
: undefined;
|
||||
await writeLocalObject(verified.b, verified.o, req, contentType);
|
||||
res.status(200).json({ ok: true });
|
||||
} catch (err) {
|
||||
req.log.error({ err }, "local storage upload failed");
|
||||
res.status(500).json({ error: "Upload failed" });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
Reference in New Issue
Block a user