DodoTransfer/lib/endpoints.js

'use strict';
const pug = require('pug');
const express = require('express');
const morgan = require('morgan');
const compression = require('compression');
const path = require('path');
const assert = require('assert');
const { createHash, randomUUID } = require('node:crypto');
const httpErrors = require('http-errors');
const config = require('../config');
const eventBus = require('./eventBus');
const tusboy = require('./tusboy');
const Store = require('./store');
const Db = require('./db');
const tusMeta = require('./tusboy/tus-metadata');
const utils = require('./utils');
const auth = require('./auth');
const settings = require('./settings');
const mime = require('./mime');
const { pipeArchive } = require('./archive');
const createAdmin = require('./admin');
const debug = require('debug')('psitransfer:main');
const { hashPassword, verifyPassword } = require('./passwordHash');

/**
 * Naming:
 * sid: Group of files (bucket)
 * key: File
 * fid: {sid}++{key}
 */

// Archive links of older versions used md5
function md5Hex(input) {
  return createHash('md5').update(input).digest('hex');
}

function sha256Hex(input) {
  return createHash('sha256').update(input).digest('hex');
}

/** Decoded path segment under the /files mount (must match req.params used by tusboy). */
function decodedUploadPathSegment(req) {
  const raw = req.path.startsWith('/') ? req.path.slice(1) : req.path;
  try {
    return decodeURIComponent(raw);
  } catch {
    return null;
  }
}

// Pages are a thin shell, the apps are rendered by the browser
const pageTemplate = pug.compileFile(path.join(__dirname, '../public/pug/page.pug'));
const errorTemplate = pug.compileFile(path.join(__dirname, '../public/pug/error.pug'));

function pageVars(req, extra = {}) {
  const brand = settings.brand();
  return {
    baseUrl: config.baseUrl,
    uploadAppPath: config.uploadAppPath || config.baseUrl,
    brand,
    // never allow "</script>" inside the inline JSON
    brandJson: JSON.stringify(brand).replace(/</g, '\\u003c'),
    lang: req.translations,
    ...extra,
  };
}

const renderPage = (req, page) => pageTemplate(pageVars(req, { page }));
const sendError = (req, res, status, error) => res.status(status).send(errorTemplate(pageVars(req, { error })));

const store = new Store(config.uploadDir);
const db = new Db(config.uploadDir, store);
db.init();
const app = express();

app.disable('x-powered-by');
app.use(compression());
app.use(express.json({ limit: '100kb' }));

if (config.accessLog) {
  app.use(morgan(config.accessLog));
}

if (config.trustProxy) {
  app.set('trust proxy', config.trustProxy);
}

if (config.forceHttps) {
  app.enable('trust proxy');
  app.use(function(req, res, next) {
    if (req.secure) return next();
    const target = config.forceHttps === 'true' ? 'https://' + req.headers.host : config.forceHttps;
    res.redirect(target + req.url);
  });
}

// Health check for Docker and monitoring: no login, no data
app.get(`${ config.baseUrl }healthz`, (req, res) => {
  res.set('Cache-Control', 'no-store');
  res.type('text/plain').send('ok');
});

// Logo and favicon: the one set on the admin page or the built-in one
app.get(`${ config.baseUrl }assets/brand-logo`, (req, res) => {
  const logo = settings.logoInfo();
  res.set({
    'Content-Type': logo.mime,
    'X-Content-Type-Options': 'nosniff',
    'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; sandbox",
    // the url carries a hash of the image (?v=), a new logo gets a new url
    'Cache-Control': 'public, max-age=86400',
  });
  // the settings folder is a hidden folder (.settings)
  res.sendFile(logo.path, { dotfiles: 'allow' });
});

// Static files
app.use(`${ config.baseUrl }app`, express.static(path.join(__dirname, '../public/app')));
app.use(`${ config.baseUrl }assets`, express.static(path.join(__dirname, '../public/assets')));

// Resolve language
app.use((req, res, next) => {
  // without the header acceptsLanguages() would answer with the first language of the list
  const lang = (req.get('accept-language') && req.acceptsLanguages(...Object.keys(config.languages)))
    || config.defaultLanguage;
  req.translations = config.languages[lang];
  next();
});

// robots.txt
app.get(`${ config.baseUrl }robots.txt`, (req, res) => {
  res.sendFile(path.join(__dirname, '../public/robots.txt'));
});

// Init plugins
config.plugins.forEach(pluginName => {
  require('../plugins/' + pluginName)(eventBus, app, config, db);
});

// Upload App
app.get(config.uploadAppPath, (req, res) => {
  res.send(renderPage(req, 'upload'));
});

// Return translations
app.get(`${ config.baseUrl }lang.json`, (req, res) => {
  eventBus.emit('getLang', req.translations);
  res.json(req.translations);
});

// Branding (name, logo, footer)
app.get(`${ config.baseUrl }brand.json`, (req, res) => {
  res.set('Cache-Control', 'no-store');
  res.json(settings.brand());
});

// Login: exchange the upload password for a token which is renewed on every use
app.post(`${ config.baseUrl }login`, (req, res) => {
  if (!config.uploadPass) return res.json({ authRequired: false });
  const password = req.body && req.body.password;
  if (typeof password !== 'string' || !auth.safeEqual(password, config.uploadPass)) {
    setTimeout(() => res.status(403).json({ message: 'Forbidden' }), 500);
    return;
  }
  const { token, exp } = auth.issue({}, config.tokenTtl);
  res.set('Cache-Control', 'no-store');
  res.json({ token, exp });
});

// Config
app.get(`${ config.baseUrl }config.json`, (req, res) => {
  // Upload password protection
  if (!auth.uploadAuth(req)) return auth.deny(req, res, 200);

  const frontendConfig = {
    retentions: config.retentions,
    quickRetentions: config.quickRetentions,
    defaultRetention: config.defaultRetention,
    mailTemplate: config.mailTemplate,
    requireBucketPassword: config.requireBucketPassword,
    maxFileSize: config.maxFileSize,
    maxBucketSize: config.maxBucketSize,
    disableQrCode: config.disableQrCode,
    authRequired: !!config.uploadPass,
  };
  // every use renews the login: the token is valid for tokenTtl from now on
  if (config.uploadPass) {
    frontendConfig.session = auth.issue({}, config.tokenTtl);
  }

  eventBus.emit('getFrontendConfig', frontendConfig);

  res.set('Cache-Control', 'no-store');
  res.json(frontendConfig);
});

// Admin: page, file management, settings
app.use(`${ config.baseUrl }admin`, createAdmin({ db, store, renderPage }));

const completeFiles = sid => (db.get(sid) || []).filter(file => !file.isPartial);
const archiveToken = files => sha256Hex(files.map(f => f.key).join()).slice(0, 32);

function publicItem(sid, item) {
  // the password hash stays on the server
  const { password, ...metadata } = item.metadata;
  const oneTime = metadata.retention === 'one-time';
  return {
    ...item,
    metadata: { ...metadata, hasPassword: !!password },
    url: `${ config.baseUrl }files/${ sid }++${ item.key }`,
    // one-time files show no content before they are downloaded
    thumbUrl: metadata.thumb && !oneTime ? `${ config.baseUrl }files/${ sid }++${ item.key }.thumb` : null,
    previewType: oneTime ? null : mime.previewKind(metadata.name),
    expiresAt: db.expiresAt(item),
  };
}

// List files / Download App
app.get(`${ config.baseUrl }:sid`, async (req, res, next) => {
  if (req.params.sid.endsWith('.json')) {
    const sid = req.params.sid.slice(0, -5);
    const files = completeFiles(sid);
    if (!files.length) return res.status(404).end();

    res.header('Cache-control', 'private, max-age=0, no-cache, no-store, must-revalidate');

    // Currently, every item in a bucket must have the same password.
    // Files uploaded with "internalBypass" are open for everyone with a valid login.
    const guarded = files.filter(item => item.metadata.password);
    const bypass = guarded.length > 0
      && guarded.every(item => item.metadata.internalBypass === 'true')
      && auth.hasSession(req);
    try {
      if (!bypass) {
        const pass = req.get('x-download-pass') || '';
        for (const item of guarded) {
          const ok = await verifyPassword(item.metadata.password, pass);
          if (!ok) {
            setTimeout(() => res.status(401).send('Unauthorized'), 500);
            return;
          }
        }
      }
    } catch (e) {
      console.error(e);
      setTimeout(() => res.status(401).send('Unauthorized'), 500);
      return;
    }

    const items = files.map(item => publicItem(sid, item));
    res.json({
      items,
      archiveToken: archiveToken(files),
      bypassed: bypass,
      oneTime: items.every(item => item.metadata.retention === 'one-time'),
      expiresAt: Math.min(...items.map(item => item.expiresAt)),
      // a valid login is renewed by using the download page as well
      session: auth.hasSession(req) ? auth.issue({}, config.tokenTtl) : null,
      config: {
        maxPreviewSize: config.maxPreviewSize
      }
    });
  } else {
    if (!completeFiles(req.params.sid).length) return next();
    res.send(renderPage(req, 'download'));
  }
});

// Thumbnail of a file
function sendThumb(req, res, fid) {
  const notFound = () => sendError(req, res, 404, 'File not found');
  if (!utils.isSafeTusUploadId(fid)) return notFound();
  const [sid, key] = fid.split('++');
  const item = completeFiles(sid).find(file => file.key === key);
  if (!item || !item.metadata.thumb || item.metadata.retention === 'one-time') return notFound();
  res.set({
    'Content-Type': item.metadata.thumb,
    'X-Content-Type-Options': 'nosniff',
    'Cache-Control': 'private, max-age=3600',
  });
  res.sendFile(store.thumbFilename(fid), err => {
    if (err && !res.headersSent) notFound();
  });
}

// Download files
app.get(`${ config.baseUrl }files/:fid`, async (req, res, next) => {
  // let tusboy handle HEAD requests with Tus Header
  if (req.method === 'HEAD' && req.get('Tus-Resumable')) return next();

  // Disable HTTP transport compression for file downloads.
  // Archives already handle their own compression (zip/gzip), and for single
  // files this preserves Content-Length and Range request support needed for
  // resumable downloads of large files.
  res.set('Cache-Control', 'no-transform');

  const sid = req.params.fid.split('++')[0];

  if (req.params.fid.endsWith('.thumb')) {
    return sendThumb(req, res, req.params.fid.slice(0, -'.thumb'.length));
  }

  // Download all files
  const archiveMatch = /^(.+)\+\+([a-f0-9]{32})\.(tar\.gz|zip)$/i.exec(req.params.fid);
  if (archiveMatch) {
    const format = archiveMatch[3] === 'zip' ? 'zip' : 'tar.gz';
    const bucket = completeFiles(sid);

    if (!bucket.length) return sendError(req, res, 404, 'Download bucket not found.');

    const keyList = bucket.map(f => f.key).join();
    const expectedLegacy = `${ sid }++${ md5Hex(keyList) }.${ format }`;
    const expectedNew = `${ sid }++${ sha256Hex(keyList).slice(0, 32) }.${ format }`;

    if (req.params.fid !== expectedLegacy && req.params.fid !== expectedNew) {
      return sendError(req, res, 404, 'Invalid link');
    }
    debug(`Download Bucket ${ sid }`);

    const isHead = req.method === 'HEAD';
    const files = bucket.slice();

    // atomically claim all one-time files; HEAD neither claims nor consumes
    const claimed = [];
    const release = () => claimed.splice(0).forEach(info => db.release(sid, info.key));
    for (const info of files.filter(info => info.metadata.retention === 'one-time')) {
      if (isHead ? db.isClaimed(sid, info.key) : !db.claim(sid, info.key)) {
        release();
        return sendError(req, res, 404, 'File not found');
      }
      if (!isHead) claimed.push(info);
    }

    const filename = `${ sid }.${ format }`;
    res.header('Content-Disposition', utils.contentDisposition(filename, 'files'));

    // release the claims on aborted downloads
    res.on('close', () => {
      if (!res.writableFinished) release();
    });

    // remove one-time files after download
    res.on('finish', async () => {
      if (isHead || res.statusCode !== 200) return release();
      let removeFailed = false;
      try {
        await Promise.all(files.map(info => info.metadata.retention === 'one-time'
          ? db.remove(info.metadata.sid, info.metadata.key)
          : db.updateLastDownload(info.metadata.sid, info.metadata.key)
        ));

        eventBus.emit('archiveDownloaded', {
          sid,
          file: filename,
          metadata: files[0].metadata,
          bucket: files,
          url: req.protocol + '://' + req.get('host') + req.originalUrl,
        });
      }
      catch (e) {
        console.error(e);
        removeFailed = true;
      }
      // keep the claims if removal failed to prevent further downloads
      if (!removeFailed) release();
    });

    return pipeArchive(res, bucket, format, store);
  }

  // Download single file
  debug(`Download ${ req.params.fid }`);
  try {
    if (req.params.fid.includes('++') && !utils.isSafeTusUploadId(req.params.fid)) {
      return sendError(req, res, 404, 'Invalid link');
    }
    const info = await store.info(req.params.fid); // throws on 404
    const isHead = req.method === 'HEAD';
    const oneTime = info.metadata.retention === 'one-time';

    // Inline preview: only for types a browser can show, never for one-time files
    // (looking at them would use them up). It is no download: not counted, no webhook.
    const inline = req.query.inline === '1';
    const previewType = inline ? mime.previewMime(info.metadata.name) : null;
    if (inline && (!previewType || oneTime)) {
      return sendError(req, res, 404, 'No preview available');
    }

    // atomically claim one-time files; HEAD neither claims nor consumes
    let claimed = false;
    if (oneTime) {
      claimed = !isHead && db.claim(info.metadata.sid, info.metadata.key);
      if (isHead ? db.isClaimed(info.metadata.sid, info.metadata.key) : !claimed) {
        return sendError(req, res, 404, 'File not found');
      }
    }
    const release = () => {
      if (!claimed) return;
      claimed = false;
      db.release(info.metadata.sid, info.metadata.key);
    };

    if (inline) {
      res.set({
        'Content-Type': previewType,
        'Content-Disposition': 'inline',
        'X-Content-Type-Options': 'nosniff',
      });
      // the sandbox keeps scripts of uploaded files from running, but it also
      // switches off the pdf viewer of the browsers
      if (previewType !== 'application/pdf') {
        res.set('Content-Security-Policy', "default-src 'none'; img-src data:; media-src 'self'; style-src 'unsafe-inline'; sandbox");
      }
    } else {
      res.set('Content-Disposition', utils.contentDisposition(info.metadata.name, info.key));
    }
    res.sendFile(store.getFilename(req.params.fid), err => {
      if (!err) return;
      release();
      if (!res.headersSent) next(err);
    });

    // release the claim on aborted downloads
    res.on('close', () => {
      if (!res.writableFinished) release();
    });

    // remove one-time files after download
    res.on('finish', async () => {
      if (isHead || inline || ![200, 206].includes(res.statusCode)) return release();
      let removeFailed = false;
      try {
        if (oneTime) {
          await db.remove(info.metadata.sid, info.metadata.key);
        } else {
          await db.updateLastDownload(info.metadata.sid, info.metadata.key);
        }

        eventBus.emit('fileDownloaded', {
          sid,
          file: info.metadata.name,
          metadata: info.metadata,
          url: req.protocol + '://' + req.get('host') + req.originalUrl,
        });
      }
      catch (e) {
        console.error(e);
        removeFailed = oneTime;
      }
      // keep the claim if removal failed to prevent further downloads
      if (!removeFailed) release();
    });
  }
  catch (e) {
    sendError(req, res, 404, e.message);
  }
});


// Upload a thumbnail for a finished upload (made by the browser of the uploader)
app.put(`${ config.uploadAppPath }files/:fid`,
  express.raw({ type: () => true, limit: '64kb' }),
  async (req, res, next) => {
    try {
      if (!auth.uploadAuth(req)) return auth.deny(req, res);
      if (!req.params.fid.endsWith('.thumb')) return res.status(404).end();
      const fid = req.params.fid.slice(0, -'.thumb'.length);
      if (!utils.isSafeTusUploadId(fid)) return res.status(400).end('Invalid upload id');
      const [sid, key] = fid.split('++');
      const item = completeFiles(sid).find(file => file.key === key);
      if (!item) return res.status(404).end('Upload not found');
      if (db.isLocked(sid)) return res.status(400).end('Bucket locked');
      const type = mime.sniffImage(req.body);
      if (!type) return res.status(400).end('Invalid thumbnail');
      await store.writeThumb(fid, req.body);
      await db.updateMetadata(sid, key, { thumb: type });
      res.status(204).end();
    } catch (e) {
      next(e);
    }
  }
);


// Upload file
app.use(`${ config.uploadAppPath }files`,
  async function(req, res, next) {
    // Upload password protection
    if (!auth.uploadAuth(req)) return auth.deny(req, res);

    if (req.method === 'GET') return res.status(405).end();

    const fid = decodedUploadPathSegment(req);
    if (fid === null) {
      return res.status(400).end('Invalid path encoding');
    }

    // Lock bucket by PATCH /files/:sid?lock=yes
    if (fid && !fid.includes('++') && req.method === 'PATCH' && req.query.lock) {
      if (!utils.isSafeBucketFid(fid)) {
        return res.status(400).end('Invalid bucket id');
      }
      await db.lock(fid);
      return res.status(204).end('Bucket locked');
    }

    if (['POST', 'PATCH'].includes(req.method)) {
      if (fid && !fid.includes('++') && !utils.isSafeBucketFid(fid)) {
        return res.status(400).end('Invalid bucket id');
      }
      if (fid && !fid.includes('++') && db.isLocked(fid)) {
        return res.status(400).end('Bucket locked');
      }
      if (fid) {
        if (fid.includes('++') && !utils.isSafeTusUploadId(fid)) {
          return res.status(400).end('Invalid upload id');
        }
        if (fid.includes('++') && db.isLocked(fid.split('++')[0])) {
          return res.status(400).end('Bucket locked');
        }
        try {
          const info = await store.info(fid);
          if (!info.isPartial) {
            return res.status(400).end('Upload already completed');
          }
        } catch (e) {
          if (!(e instanceof httpErrors.NotFound)) {
            console.error(e);
            return next(e);
          }
        }
      }
    }

    if (req.method === 'POST') {
      // validate meta-data
      // !! tusMeta.encode supports only strings !!
      const meta = tusMeta.decode(req.get('Upload-Metadata'));

      try {
        assert(meta.name, 'tus meta prop missing: name');
        assert(meta.sid, 'tus meta prop missing: sid');
        if (!utils.isSafeBasename(meta.sid)) {
          return res.status(400).end('Invalid bucket id');
        }
        assert(meta.retention, 'tus meta prop missing: retention');
        assert(Object.keys(config.retentions).indexOf(meta.retention) >= 0,
          `invalid tus meta prop retention. Value ${ meta.retention } not in [${ Object.keys(config.retentions).join(',') }]`);

        // Prevent ZipSlip/tar path traversal by requiring a safe basename at upload time.
        // Policy (flat archive): no directories, no absolute paths, no traversal, no control chars.
        if (!utils.isSafeBasename(meta.name)) {
          return res.status(400).end('Invalid file name');
        }

        const uploadLength = req.get('Upload-Length');
        assert(uploadLength, 'missing Upload-Length header');

        // Restrict creating new files for locked buckets
        if(db.isLocked(meta.sid)) {
          return res.status(400).end('Bucket locked');
        }

        // these are set by the server only
        for (const reserved of ['thumb', 'lastDownload', 'bucketLocked', 'buckedLocked']) {
          delete meta[reserved];
        }

        const password = typeof meta.password === 'string' && meta.password.length > 0 ? meta.password : '';

        // A password is mandatory, except for one-time downloads
        if (config.requireBucketPassword && meta.retention !== 'one-time' && !password) {
          return res.status(400).json({ message: 'A password is required for this retention.' });
        }

        meta.uploadLength = uploadLength;
        meta.key = randomUUID();
        meta.createdAt = Date.now().toString();

        // limit file and bucket size
        if (config.maxFileSize && config.maxFileSize < +uploadLength) {
          return res
            .status(413)
            .json({ message: `File exceeds maximum upload size ${ config.maxFileSize }.` });
        } else if (config.maxBucketSize && db.bucketSize(meta.sid) + +uploadLength > config.maxBucketSize) {
          return res
            .status(413)
            .json({ message: `Bucket exceeds maximum upload size ${ config.maxBucketSize }.` });
        }

        // store changed metadata for tusboy
        if (password) {
          meta.password = await hashPassword(password);
          // logged in users may skip the password, only meaningful with a login
          if (meta.internalBypass === 'true' && config.uploadPass) meta.internalBypass = 'true';
          else delete meta.internalBypass;
        } else {
          delete meta.password;
          delete meta.internalBypass;
        }
        req.headers['upload-metadata'] = tusMeta.encode(meta);
        // for tusboy getKey()
        req.FID = meta.sid + '++' + meta.key;

        db.add(meta.sid, meta.key, {
          "isPartial": true,
          metadata: meta
        });
      }
      catch (e) {
        console.error(e);
        return res.status(400).end(e.message);
      }
    }

    next();
  },

  // let tusboy handle the upload
  tusboy(store, {
    getKey: req => req.FID,
    maxUploadLength: config.maxFileSize || Infinity,
    afterComplete: (req, upload, fid) => {
      db.add(upload.metadata.sid, upload.metadata.key, upload);
      debug(`Completed upload ${ fid }, size=${ upload.size } name=${ upload.metadata.name }`);
      eventBus.emit('fileUploaded', upload);
    },
  })
);

app.use((req, res, next) => {
  if (req.url === config.baseUrl) {
    return res.redirect(config.uploadAppPath);
  }

  sendError(req, res, 404, 'Download bucket not found.');
});

module.exports = app;