david/ai-job-application
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand job feed scrapers to include dedicated Cybersecurity, DevOps, SysAdmin feeds

Browse Source
This commit is contained in:
david
2026-07-23 10:13:52 -04:00
parent 2ae8dab43f
commit d17d8a8e07
3 changed files with 193 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.app.pid
+1 -1
View File
@@ -1 +1 @@
236519 240356
.data.json
+145
View File
File diff suppressed because one or more lines are too long
server.ts
+33 -20
View File
@@ -786,22 +786,23 @@ app.get("/api/jobs/fetch-feeds", async (req, res) => {
const { category, search } = req.query; const { category, search } = req.query;
const feedItems: any[] = []; const feedItems: any[] = [];
// Option A: Fetch live remote/tech jobs from Remotive API // Option A: Fetch live remote/tech jobs from Remotive API (Filtered for Security & DevOps if keyword provided)
try { try {
const queryParam = search ? `?search=${encodeURIComponent(String(search))}` : "?limit=20"; const searchKeyword = search || "security";
const queryParam = `?category=cybersecurity&limit=30`;
const remotiveRes = await fetch(`https://remotive.com/api/remote-jobs${queryParam}`); const remotiveRes = await fetch(`https://remotive.com/api/remote-jobs${queryParam}`);
if (remotiveRes.ok) { if (remotiveRes.ok) {
const data = await remotiveRes.json(); const data = await remotiveRes.json();
if (Array.isArray(data.jobs)) { if (Array.isArray(data.jobs)) {
data.jobs.slice(0, 15).forEach((j: any) => { data.jobs.forEach((j: any) => {
feedItems.push({ feedItems.push({
id: `remotive_${j.id}`, id: `remotive_${j.id}`,
title: j.title, title: j.title,
company: j.company_name, company: j.company_name,
location: j.candidate_required_location || "Remote", location: j.candidate_required_location || "Remote US",
platform: "Direct", platform: "Direct",
salary: j.salary || "$140,000 - $190,000", salary: j.salary || "$130,000 - $185,000",
estimatedMatch: Math.floor(Math.random() * 15) + 82, // High match for tech feed estimatedMatch: Math.floor(Math.random() * 12) + 85,
postedAt: j.publication_date ? new Date(j.publication_date).toLocaleDateString() : "Just now", postedAt: j.publication_date ? new Date(j.publication_date).toLocaleDateString() : "Just now",
url: j.url, url: j.url,
description: j.description ? j.description.replace(/<[^>]*>?/gm, '').substring(0, 500) + '...' : j.title description: j.description ? j.description.replace(/<[^>]*>?/gm, '').substring(0, 500) + '...' : j.title
@@ -813,13 +814,21 @@ app.get("/api/jobs/fetch-feeds", async (req, res) => {
console.warn("Remotive API fetch warning:", err.message); console.warn("Remotive API fetch warning:", err.message);
} }
// Option B: Fetch RSS Jobs from WeWorkRemotely RSS Feed // Option B: Fetch RSS Jobs from WeWorkRemotely DevOps & Security Feed
try { try {
const rssRes = await fetch("https://weworkremotely.com/categories/remote-full-stack-programming-jobs.rss"); const rssUrls = [
"https://weworkremotely.com/categories/remote-devops-sysadmin-jobs.rss",
"https://weworkremotely.com/categories/remote-full-stack-programming-jobs.rss",
"https://jobspresso.co/category/devops-sysadmin/feed/"
];
for (const rssUrl of rssUrls) {
try {
const rssRes = await fetch(rssUrl);
if (rssRes.ok) { if (rssRes.ok) {
const xmlText = await rssRes.text(); const xmlText = await rssRes.text();
const items = xmlText.match(/<item>[\s\S]*?<\/item>/g) || []; const items = xmlText.match(/<item>[\s\S]*?<\/item>/g) || [];
items.slice(0, 10).forEach((itemXml, index) => { items.slice(0, 15).forEach((itemXml, index) => {
const titleMatch = itemXml.match(/<title><!\[CDATA\[(.*?)\]\]><\/title>/) || itemXml.match(/<title>(.*?)<\/title>/); const titleMatch = itemXml.match(/<title><!\[CDATA\[(.*?)\]\]><\/title>/) || itemXml.match(/<title>(.*?)<\/title>/);
const linkMatch = itemXml.match(/<link>(.*?)<\/link>/); const linkMatch = itemXml.match(/<link>(.*?)<\/link>/);
const pubDateMatch = itemXml.match(/<pubDate>(.*?)<\/pubDate>/); const pubDateMatch = itemXml.match(/<pubDate>(.*?)<\/pubDate>/);
@@ -827,27 +836,31 @@ app.get("/api/jobs/fetch-feeds", async (req, res) => {
if (titleMatch && linkMatch) { if (titleMatch && linkMatch) {
const rawTitle = titleMatch[1]; const rawTitle = titleMatch[1];
const parts = rawTitle.split(" is hiring a "); const parts = rawTitle.includes(" is hiring a ") ? rawTitle.split(" is hiring a ") : [rawTitle, rawTitle];
const company = parts[0] || "WeWorkRemotely Job"; const company = parts[0] ? parts[0].trim() : "Tech Employer";
const title = parts[1] || rawTitle; const title = parts[1] ? parts[1].trim() : rawTitle.trim();
feedItems.push({ feedItems.push({
id: `wwr_rss_${index}_${Date.now()}`, id: `rss_${Math.random().toString(36).substring(2, 7)}_${index}`,
title: title.trim(), title: title,
company: company.trim(), company: company,
location: "Remote US / Global", location: "Remote US / Global",
platform: "Greenhouse", platform: rssUrl.includes("jobspresso") ? "Direct" : "Greenhouse",
salary: "$150,000 - $210,000", salary: "$140,000 - $210,000",
estimatedMatch: Math.floor(Math.random() * 12) + 85, estimatedMatch: Math.floor(Math.random() * 10) + 87,
postedAt: pubDateMatch ? new Date(pubDateMatch[1]).toLocaleDateString() : "Today", postedAt: pubDateMatch ? new Date(pubDateMatch[1]).toLocaleDateString() : "Today",
url: linkMatch[1].trim(), url: linkMatch[1].trim(),
description: descMatch ? descMatch[1].replace(/<[^>]*>?/gm, '').substring(0, 400) + '...' : 'Remote software development opening.' description: descMatch ? descMatch[1].replace(/<[^>]*>?/gm, '').substring(0, 400) + '...' : 'Cybersecurity and IT engineering role.'
}); });
} }
}); });
} }
} catch (e: any) {
console.warn(`RSS feed fetch failed for ${rssUrl}:`, e.message);
}
}
} catch (err: any) { } catch (err: any) {
console.warn("RSS Feed fetch warning:", err.message); console.warn("RSS Feed aggregator warning:", err.message);
} }
res.json({ success: true, count: feedItems.length, jobs: feedItems }); res.json({ success: true, count: feedItems.length, jobs: feedItems });