/**
* VidGap Scraper - Multi-Platform Downloader (TikTok, Douyin, Lemon8)
* Author: Vamz Spectre (VamxAPI)
* Target: vidgap.com Next.js App Router Server Actions + WebSocket Engine (socket.vidgap.com)
*
* Fitur:
* - Auto-Healing Dynamic Action IDs: Otomatis mendeteksi hash Server Action Next.js terbaru
* sehingga ANTI-404 bahkan saat VidGap melakukan deploy build baru!
* - Dynamic Page Session Management: Menangani autentikasi sesi JWT (vg_session) otomatis
* - Zero Configuration: Cukup input link/teks, semua dideteksi otomatis (Platform, Jenis Konten, Tab)
* - Dukungan penuh platform:
* - TikTok (Video, Foto/Slideshow, Audio/Musik, Story, Profil @user)
* - Douyin (抖音 - Video, Gambar, Musik, Profil)
* - Lemon8 (Postingan, Galeri Gambar)
* - Auto-clean & extraction link dari teks acak / hasil share WhatsApp, Telegram, Clipboard
* - Mengambil seluruh metadata dan media:
* - Video HD tanpa watermark (langsung CDN)
* - Video Standar (Non-watermark)
* - Video Watermark (jika ada)
* - Ukuran file terformat (MB/KB) & raw bytes
* - Audio / Musik MP3 lengkap (Judul, durasi, URL file)
* - Slideshow galeri gambar beresolusi tinggi lengkap
* - Thumbnail, Cover, Dynamic Animated Cover
* - Informasi Author/Kreator (Username, Nickname, Avatar HD, Profile URL, ID)
* - Statistik lengkap (Views, Likes, Comments, Shares) + angka terformat (1.2K, 3.4M)
* - Hashtag yang diekstrak dari caption
* - Support direct cache extraction & WebSocket live job queue (socket.vidgap.com)
* - Resolusi otomatis redirect token (/api/video/redirect/...) menjadi URL CDN langsung siap unduh
* - Siap digunakan via CLI (node vidgap.js ) maupun diimpor sebagai modul (CommonJS)
*/
const dns = require("dns");
if (dns && typeof dns.setDefaultResultOrder === "function") {
dns.setDefaultResultOrder("ipv4first");
}
const { io } = require("socket.io-client");
const VIDGAP_BASE_URL = "https://vidgap.com";
const VIDGAP_SOCKET_URL = "https://socket.vidgap.com";
const DEFAULT_USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
// Cache in-memory untuk Server Action IDs dengan fallback hash Next.js build saat ini
let cachedActionIds = {
findContent: "4091bfdbb5d988e317bc1807ecf621788a15b8b1e8",
updateToken: "400e140eaf0c4c29003d6051a6b2aaf271eb2eaf4a",
lastFetched: 0,
};
// Cache in-memory untuk page-session cookie (vg_session)
let cachedSessionCookie = null;
/**
* Mendapatkan dan me-refresh cookie session VidGap (vg_session)
* @param {boolean} forceRefresh
* @returns {Promise}
*/
async function getPageSessionCookie(forceRefresh = false) {
if (!forceRefresh && cachedSessionCookie && cachedSessionCookie.expiresAt > Date.now()) {
return cachedSessionCookie.cookie;
}
try {
const res = await fetch(`${VIDGAP_BASE_URL}/api/page-session`, {
method: "POST",
headers: {
"User-Agent": DEFAULT_USER_AGENT,
Origin: VIDGAP_BASE_URL,
Referer: `${VIDGAP_BASE_URL}/`,
},
});
const setCookies =
typeof res.headers.getSetCookie === "function"
? res.headers.getSetCookie()
: [res.headers.get("set-cookie") || ""];
const vgSession = setCookies
.map((c) => (c ? c.split(";")[0].trim() : ""))
.find((c) => c && c.startsWith("vg_session="));
if (vgSession) {
cachedSessionCookie = {
cookie: vgSession,
expiresAt: Date.now() + 15 * 60 * 1000,
};
return vgSession;
}
} catch {}
return cachedSessionCookie?.cookie || "";
}
/**
* Mendapatkan Server Action ID terbaru secara dinamis dari bundle Next.js VidGap
* @param {boolean} forceRefresh
* @returns {Promise<{findContent: string, updateToken: string}>}
*/
async function getActionIds(forceRefresh = false) {
if (
!forceRefresh &&
cachedActionIds.findContent &&
cachedActionIds.updateToken &&
Date.now() - cachedActionIds.lastFetched < 3600000
) {
return cachedActionIds;
}
try {
const htmlRes = await fetch(`${VIDGAP_BASE_URL}/`, {
headers: { "User-Agent": DEFAULT_USER_AGENT },
});
const html = await htmlRes.text();
const chunkUrls = [...html.matchAll(/src="(\/_next\/static\/chunks\/[^"]+)"/g)].map((m) => m[1]);
let findContentId = null;
let updateTokenId = null;
for (const chunkPath of chunkUrls) {
try {
const chunkRes = await fetch(`${VIDGAP_BASE_URL}${chunkPath}`, {
headers: { "User-Agent": DEFAULT_USER_AGENT },
});
const chunkText = await chunkRes.text();
if (!findContentId && chunkText.includes("findContentActionsQueue")) {
const match = chunkText.match(/createServerReference\)\("([a-f0-9]+)"[^)]*"findContentActionsQueue"\)/);
if (match) findContentId = match[1];
}
if (!updateTokenId && chunkText.includes("updateDownloadToken")) {
const match = chunkText.match(/createServerReference\)\("([a-f0-9]+)"[^)]*"updateDownloadToken"\)/);
if (match) updateTokenId = match[1];
}
if (findContentId && updateTokenId) break;
} catch {}
}
if (findContentId) cachedActionIds.findContent = findContentId;
if (updateTokenId) cachedActionIds.updateToken = updateTokenId;
cachedActionIds.lastFetched = Date.now();
} catch {}
return cachedActionIds;
}
/**
* Ekstrak dan normalisasi URL dari input teks mentah (misal teks share WA/Telegram)
* @param {string} rawInput - Teks atau URL mentah
* @returns {string} Clean URL
*/
function extractAndNormalize(rawInput) {
if (!rawInput || typeof rawInput !== "string") {
throw new Error("Parameter link atau URL wajib diisi.");
}
const trimmed = rawInput.trim();
// 1. Ekstrak URL http/https dari teks
const urlRegex = /(https?:\/\/[^\s<>"{}|\\^~\[\]`]+)/i;
const match = trimmed.match(urlRegex);
let detectedUrl = match ? match[1] : null;
if (detectedUrl) {
// Bersihkan karakter tanda baca penutup yang sering menempel
detectedUrl = detectedUrl.replace(/[)\]}>,.;!]+$/, "");
return detectedUrl;
}
// 2. Jika bukan link http(s), cek format username atau ID
if (trimmed.startsWith("@")) {
return `https://www.tiktok.com/@${trimmed.slice(1).trim().toLowerCase()}`;
}
// Contoh: 19 digit angka (TikTok video ID murni)
if (/^\d{15,21}$/.test(trimmed)) {
return `https://www.tiktok.com/@user/video/${trimmed}`;
}
// Contoh: username biasa tanpa @ (misal: "tiktok")
if (/^[a-zA-Z0-9_.-]{3,30}$/.test(trimmed) && !trimmed.includes("/") && !trimmed.includes(".")) {
return `https://www.tiktok.com/@${trimmed.toLowerCase()}`;
}
return trimmed;
}
/**
* Mendeteksi platform target otomatis berdasarkan URL
* @param {string} url
* @returns {"douyin" | "lemon8" | "tiktok"}
*/
function detectPlatform(url) {
const lower = String(url || "").toLowerCase();
if (lower.includes("douyin.com") || lower.includes("iesdouyin.com")) {
return "douyin";
}
if (
lower.includes("lemon8-app.com") ||
lower.includes("lemon8.com") ||
lower.includes("s.lemon8-app.com")
) {
return "lemon8";
}
return "tiktok";
}
/**
* Mendeteksi tab / tipe konten secara otomatis berdasarkan pola link
* @param {string} url
* @returns {"all" | "audio" | "slideshow" | "profile" | "story"}
*/
function detectTab(url) {
const lower = String(url || "").toLowerCase();
if (lower.includes("/music/")) return "audio";
if (lower.includes("/photo/") || lower.includes("slideshow")) return "slideshow";
if (lower.includes("/story/")) return "story";
if (
/@[\w.-]+(?:\/?$|\?[^/]*$)/.test(lower) &&
!lower.includes("/video/") &&
!lower.includes("/photo/")
) {
return "profile";
}
return "all";
}
/**
* Helper format ukuran bytes ke format yang mudah dibaca (KB / MB / GB)
* @param {number} bytes
* @returns {string | undefined}
*/
function formatBytes(bytes) {
if (!bytes || isNaN(bytes) || bytes <= 0) return undefined;
const k = 1024;
const sizes = ["Bytes", "KB", "MB", "GB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
}
/**
* Helper format angka statistik ke format ringkas (1.2K, 3.4M)
* @param {number} num
* @returns {string | undefined}
*/
function formatNumber(num) {
if (typeof num !== "number" || isNaN(num)) return undefined;
if (num >= 1000000000) return (num / 1000000000).toFixed(1).replace(/\.0$/, "") + "B";
if (num >= 1000000) return (num / 1000000).toFixed(1).replace(/\.0$/, "") + "M";
if (num >= 1000) return (num / 1000).toFixed(1).replace(/\.0$/, "") + "K";
return num.toString();
}
/**
* Helper format durasi detik ke MM:SS
* @param {number} sec
* @returns {string | undefined}
*/
function formatDuration(sec) {
if (typeof sec !== "number" || isNaN(sec) || sec < 0) return undefined;
const m = Math.floor(sec / 60);
const s = Math.floor(sec % 60);
return `${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
}
/**
* Helper ekstrak hashtag dari teks judul / caption
* @param {string} text
* @returns {string[]}
*/
function extractHashtags(text) {
if (!text || typeof text !== "string") return [];
const matches = text.match(/#[\p{L}\p{N}_-]+/gu);
return matches ? [...new Set(matches)] : [];
}
/**
* Memanggil Server Action Next.js findContentActionsQueue dengan auto-healing & dynamic session
*/
async function callFindContentAction(url, platform = "tiktok", tab = "default", isRetry = false) {
const actionIds = await getActionIds(isRetry);
const vgSession = await getPageSessionCookie(isRetry);
const actionPayload = [
{
vLink: url,
agent: "Web",
tab: tab || "default",
amplified: false,
platform,
skipVgShareAccessIncrement: undefined,
},
];
const pagePath = platform === "tiktok" ? "" : platform;
const endpointUrl = `${VIDGAP_BASE_URL}/${pagePath}`;
const headers = {
"Next-Action": actionIds.findContent,
"Content-Type": "text/plain;charset=UTF-8",
"User-Agent": DEFAULT_USER_AGENT,
Origin: VIDGAP_BASE_URL,
Referer: endpointUrl,
Accept: "text/x-component",
};
if (vgSession) {
headers["Cookie"] = vgSession;
}
const res = await fetch(endpointUrl, {
method: "POST",
headers,
body: JSON.stringify(actionPayload),
});
if ((res.status === 404 || res.headers.get("x-nextjs-action-not-found")) && !isRetry) {
return callFindContentAction(url, platform, tab, true);
}
if (!res.ok) {
throw new Error(`VidGap Server Action HTTP ${res.status}: ${res.statusText}`);
}
const text = await res.text();
let sessionId = null;
let cachedData = null;
for (const line of text.split("\n")) {
const colonIdx = line.indexOf(":");
if (colonIdx === -1) continue;
const jsonStr = line.slice(colonIdx + 1).trim();
if (!jsonStr.startsWith("{") && !jsonStr.startsWith("[")) continue;
try {
const parsed = JSON.parse(jsonStr);
if (parsed?.reason === "no_page_session" && !isRetry) {
await getPageSessionCookie(true);
return callFindContentAction(url, platform, tab, true);
}
if (parsed?.sessionId) {
sessionId = parsed.sessionId;
}
if (parsed?.cached && parsed?.data) {
cachedData = parsed.data;
break;
}
if (parsed?.title && parsed?.title !== "#NO_CONTENT#") {
cachedData = parsed;
break;
}
} catch {}
}
return { sessionId, cachedData };
}
/**
* Menghubungkan ke WebSocket socket.vidgap.com untuk mengambil data jika antrean aktif
*/
async function waitForSocketData(sessionId, timeoutMs = 25000) {
return new Promise((resolve, reject) => {
let resolved = false;
const timer = setTimeout(() => {
if (resolved) return;
resolved = true;
socket.disconnect();
reject(new Error(`Timeout (${timeoutMs / 1000}s) saat menunggu data dari antrean VidGap socket.`));
}, timeoutMs);
const socket = io(VIDGAP_SOCKET_URL, {
transports: ["websocket"],
timeout: 10000,
forceNew: true,
reconnection: false,
extraHeaders: {
"User-Agent": DEFAULT_USER_AGENT,
Origin: VIDGAP_BASE_URL,
},
});
socket.on("connect", () => {
socket.emit("authenticate", sessionId);
});
socket.on("authenticate", (authRes) => {
if (authRes && authRes.status === "failed") {
clearTimeout(timer);
resolved = true;
socket.disconnect();
reject(new Error("Autentikasi sesi VidGap gagal: " + (authRes.message || "Unknown error")));
}
});
socket.on("retrieve-data", (rawData) => {
clearTimeout(timer);
if (resolved) return;
resolved = true;
socket.disconnect();
try {
const parsed = typeof rawData === "string" ? JSON.parse(rawData) : rawData;
if (!parsed) {
return reject(new Error("Data kosong diterima dari VidGap."));
}
if (parsed.errorMessage) {
return reject(new Error(parsed.errorMessage?.message || parsed.errorMessage?.type || "VidGap error"));
}
resolve(parsed.data || parsed);
} catch (err) {
reject(err);
}
});
socket.on("error", (err) => {
clearTimeout(timer);
if (resolved) return;
resolved = true;
socket.disconnect();
reject(new Error(`Koneksi socket error: ${err}`));
});
socket.on("connect_error", (err) => {
clearTimeout(timer);
if (resolved) return;
resolved = true;
socket.disconnect();
reject(new Error(`Koneksi socket connect_error: ${err?.message || err}`));
});
});
}
/**
* Mengambil cookies token unduhan dari Server Action updateDownloadToken dengan auto-healing
*/
async function obtainDownloadCookies(contentId, isRetry = false) {
if (!contentId) return "";
const actionIds = await getActionIds(isRetry);
const vgSession = await getPageSessionCookie(isRetry);
try {
const headers = {
"Next-Action": actionIds.updateToken,
"Content-Type": "text/plain;charset=UTF-8",
"User-Agent": DEFAULT_USER_AGENT,
Origin: VIDGAP_BASE_URL,
Referer: `${VIDGAP_BASE_URL}/`,
};
if (vgSession) {
headers["Cookie"] = vgSession;
}
const res = await fetch(`${VIDGAP_BASE_URL}/`, {
method: "POST",
headers,
body: JSON.stringify([String(contentId)]),
});
if ((res.status === 404 || res.headers.get("x-nextjs-action-not-found")) && !isRetry) {
return obtainDownloadCookies(contentId, true);
}
const setCookies =
typeof res.headers.getSetCookie === "function"
? res.headers.getSetCookie()
: [res.headers.get("set-cookie") || ""];
return setCookies
.filter(Boolean)
.map((c) => c.split(";")[0].trim())
.join("; ");
} catch {
return "";
}
}
/**
* Menyelesaikan URL redirect /api/video/redirect/... menjadi tautan file langsung
*/
async function resolveRedirectUrl(relativeOrFullUrl, cookieHeader) {
if (!relativeOrFullUrl || typeof relativeOrFullUrl !== "string") return undefined;
if (!relativeOrFullUrl.includes("/api/video/redirect/")) {
return relativeOrFullUrl;
}
const fullUrl = relativeOrFullUrl.startsWith("http")
? relativeOrFullUrl
: `${VIDGAP_BASE_URL}${relativeOrFullUrl.startsWith("/") ? "" : "/"}${relativeOrFullUrl}`;
try {
const res = await fetch(fullUrl, {
method: "GET",
redirect: "manual",
headers: {
Cookie: cookieHeader || "",
"User-Agent": DEFAULT_USER_AGENT,
Referer: `${VIDGAP_BASE_URL}/`,
},
});
const location = res.headers.get("location");
if (location && location.startsWith("http")) {
return location;
}
} catch {}
return fullUrl;
}
/**
* Scraper Utama VidGap (Cukup masukkan link, semua dideteksi otomatis)
* @param {string} input - Tautan, share text, atau ID TikTok/Douyin/Lemon8
* @param {object} [options] - Opsi tambahan (opsional)
* @param {"tiktok" | "douyin" | "lemon8"} [options.platform] - Override platform jika diperlukan
* @param {"all" | "video" | "audio" | "slideshow" | "story" | "profile"} [options.tab] - Override tab
* @param {number} [options.timeout] - Timeout socket dalam milidetik (default: 25000)
* @returns {Promise