david/ai-job-application
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand job feed scrapers to include dedicated Cybersecurity, DevOps, SysAdmin feeds

Browse Source
This commit is contained in:
david
2026-07-23 10:13:52 -04:00
parent 2ae8dab43f
commit d17d8a8e07
3 changed files with 193 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.app.pid
+1 -1
View File
@@ -1 +1 @@
236519 240356
.data.json
+145
View File
File diff suppressed because one or more lines are too long
server.ts
+47 -34
View File
@@ -786,22 +786,23 @@ app.get("/api/jobs/fetch-feeds", async (req, res) => {
const { category, search } = req.query; const { category, search } = req.query;
const feedItems: any[] = []; const feedItems: any[] = [];
// Option A: Fetch live remote/tech jobs from Remotive API // Option A: Fetch live remote/tech jobs from Remotive API (Filtered for Security & DevOps if keyword provided)
try { try {
const queryParam = search ? `?search=${encodeURIComponent(String(search))}` : "?limit=20"; const searchKeyword = search || "security";
const queryParam = `?category=cybersecurity&limit=30`;
const remotiveRes = await fetch(`https://remotive.com/api/remote-jobs${queryParam}`); const remotiveRes = await fetch(`https://remotive.com/api/remote-jobs${queryParam}`);
if (remotiveRes.ok) { if (remotiveRes.ok) {
const data = await remotiveRes.json(); const data = await remotiveRes.json();
if (Array.isArray(data.jobs)) { if (Array.isArray(data.jobs)) {
data.jobs.slice(0, 15).forEach((j: any) => { data.jobs.forEach((j: any) => {
feedItems.push({ feedItems.push({
id: `remotive_${j.id}`, id: `remotive_${j.id}`,
title: j.title, title: j.title,
company: j.company_name, company: j.company_name,
location: j.candidate_required_location || "Remote", location: j.candidate_required_location || "Remote US",
platform: "Direct", platform: "Direct",
salary: j.salary || "$140,000 - $190,000", salary: j.salary || "$130,000 - $185,000",
estimatedMatch: Math.floor(Math.random() * 15) + 82, // High match for tech feed estimatedMatch: Math.floor(Math.random() * 12) + 85,
postedAt: j.publication_date ? new Date(j.publication_date).toLocaleDateString() : "Just now", postedAt: j.publication_date ? new Date(j.publication_date).toLocaleDateString() : "Just now",
url: j.url, url: j.url,
description: j.description ? j.description.replace(/<[^>]*>?/gm, '').substring(0, 500) + '...' : j.title description: j.description ? j.description.replace(/<[^>]*>?/gm, '').substring(0, 500) + '...' : j.title
@@ -813,41 +814,53 @@ app.get("/api/jobs/fetch-feeds", async (req, res) => {
console.warn("Remotive API fetch warning:", err.message); console.warn("Remotive API fetch warning:", err.message);
} }
// Option B: Fetch RSS Jobs from WeWorkRemotely RSS Feed // Option B: Fetch RSS Jobs from WeWorkRemotely DevOps & Security Feed
try { try {
const rssRes = await fetch("https://weworkremotely.com/categories/remote-full-stack-programming-jobs.rss"); const rssUrls = [
if (rssRes.ok) { "https://weworkremotely.com/categories/remote-devops-sysadmin-jobs.rss",
const xmlText = await rssRes.text(); "https://weworkremotely.com/categories/remote-full-stack-programming-jobs.rss",
const items = xmlText.match(/<item>[\s\S]*?<\/item>/g) || []; "https://jobspresso.co/category/devops-sysadmin/feed/"
items.slice(0, 10).forEach((itemXml, index) => { ];
const titleMatch = itemXml.match(/<title><!\[CDATA\[(.*?)\]\]><\/title>/) || itemXml.match(/<title>(.*?)<\/title>/);
const linkMatch = itemXml.match(/<link>(.*?)<\/link>/);
const pubDateMatch = itemXml.match(/<pubDate>(.*?)<\/pubDate>/);
const descMatch = itemXml.match(/<description><!\[CDATA\[(.*?)\]\]><\/description>/);
if (titleMatch && linkMatch) { for (const rssUrl of rssUrls) {
const rawTitle = titleMatch[1]; try {
const parts = rawTitle.split(" is hiring a "); const rssRes = await fetch(rssUrl);
const company = parts[0] || "WeWorkRemotely Job"; if (rssRes.ok) {
const title = parts[1] || rawTitle; const xmlText = await rssRes.text();
const items = xmlText.match(/<item>[\s\S]*?<\/item>/g) || [];
items.slice(0, 15).forEach((itemXml, index) => {
const titleMatch = itemXml.match(/<title><!\[CDATA\[(.*?)\]\]><\/title>/) || itemXml.match(/<title>(.*?)<\/title>/);
const linkMatch = itemXml.match(/<link>(.*?)<\/link>/);
const pubDateMatch = itemXml.match(/<pubDate>(.*?)<\/pubDate>/);
const descMatch = itemXml.match(/<description><!\[CDATA\[(.*?)\]\]><\/description>/);
feedItems.push({ if (titleMatch && linkMatch) {
id: `wwr_rss_${index}_${Date.now()}`, const rawTitle = titleMatch[1];
title: title.trim(), const parts = rawTitle.includes(" is hiring a ") ? rawTitle.split(" is hiring a ") : [rawTitle, rawTitle];
company: company.trim(), const company = parts[0] ? parts[0].trim() : "Tech Employer";
location: "Remote US / Global", const title = parts[1] ? parts[1].trim() : rawTitle.trim();
platform: "Greenhouse",
salary: "$150,000 - $210,000", feedItems.push({
estimatedMatch: Math.floor(Math.random() * 12) + 85, id: `rss_${Math.random().toString(36).substring(2, 7)}_${index}`,
postedAt: pubDateMatch ? new Date(pubDateMatch[1]).toLocaleDateString() : "Today", title: title,
url: linkMatch[1].trim(), company: company,
description: descMatch ? descMatch[1].replace(/<[^>]*>?/gm, '').substring(0, 400) + '...' : 'Remote software development opening.' location: "Remote US / Global",
platform: rssUrl.includes("jobspresso") ? "Direct" : "Greenhouse",
salary: "$140,000 - $210,000",
estimatedMatch: Math.floor(Math.random() * 10) + 87,
postedAt: pubDateMatch ? new Date(pubDateMatch[1]).toLocaleDateString() : "Today",
url: linkMatch[1].trim(),
description: descMatch ? descMatch[1].replace(/<[^>]*>?/gm, '').substring(0, 400) + '...' : 'Cybersecurity and IT engineering role.'
});
}
}); });
} }
}); } catch (e: any) {
console.warn(`RSS feed fetch failed for ${rssUrl}:`, e.message);
}
} }
} catch (err: any) { } catch (err: any) {
console.warn("RSS Feed fetch warning:", err.message); console.warn("RSS Feed aggregator warning:", err.message);
} }
res.json({ success: true, count: feedItems.length, jobs: feedItems }); res.json({ success: true, count: feedItems.length, jobs: feedItems });