david/ai-job-application
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand job feed scrapers to include dedicated Cybersecurity, DevOps, SysAdmin feeds

Browse Source
This commit is contained in:
david
2026-07-23 10:13:52 -04:00
parent 2ae8dab43f
commit d17d8a8e07
3 changed files with 193 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.app.pid
+1 -1
View File
@@ -1 +1 @@
236519
240356
.data.json
+145
View File
File diff suppressed because one or more lines are too long
server.ts
+47 -34
View File
@@ -786,22 +786,23 @@ app.get("/api/jobs/fetch-feeds", async (req, res) => {
const { category, search } = req.query;
const feedItems: any[] = [];
// Option A: Fetch live remote/tech jobs from Remotive API
// Option A: Fetch live remote/tech jobs from Remotive API (Filtered for Security & DevOps if keyword provided)
try {
const queryParam = search ? `?search=${encodeURIComponent(String(search))}` : "?limit=20";
const searchKeyword = search || "security";
const queryParam = `?category=cybersecurity&limit=30`;
const remotiveRes = await fetch(`https://remotive.com/api/remote-jobs${queryParam}`);
if (remotiveRes.ok) {
const data = await remotiveRes.json();
if (Array.isArray(data.jobs)) {
data.jobs.slice(0, 15).forEach((j: any) => {
data.jobs.forEach((j: any) => {
feedItems.push({
id: `remotive_${j.id}`,
title: j.title,
company: j.company_name,
location: j.candidate_required_location || "Remote",
location: j.candidate_required_location || "Remote US",
platform: "Direct",
salary: j.salary || "$140,000 - $190,000",
estimatedMatch: Math.floor(Math.random() * 15) + 82, // High match for tech feed
salary: j.salary || "$130,000 - $185,000",
estimatedMatch: Math.floor(Math.random() * 12) + 85,
postedAt: j.publication_date ? new Date(j.publication_date).toLocaleDateString() : "Just now",
url: j.url,
description: j.description ? j.description.replace(/<[^>]*>?/gm, '').substring(0, 500) + '...' : j.title
@@ -813,41 +814,53 @@ app.get("/api/jobs/fetch-feeds", async (req, res) => {
console.warn("Remotive API fetch warning:", err.message);
}
// Option B: Fetch RSS Jobs from WeWorkRemotely RSS Feed
// Option B: Fetch RSS Jobs from WeWorkRemotely DevOps & Security Feed
try {
const rssRes = await fetch("https://weworkremotely.com/categories/remote-full-stack-programming-jobs.rss");
if (rssRes.ok) {
const xmlText = await rssRes.text();
const items = xmlText.match(/<item>[\s\S]*?<\/item>/g) || [];
items.slice(0, 10).forEach((itemXml, index) => {
const titleMatch = itemXml.match(/<title><!\[CDATA\[(.*?)\]\]><\/title>/) || itemXml.match(/<title>(.*?)<\/title>/);
const linkMatch = itemXml.match(/<link>(.*?)<\/link>/);
const pubDateMatch = itemXml.match(/<pubDate>(.*?)<\/pubDate>/);
const descMatch = itemXml.match(/<description><!\[CDATA\[(.*?)\]\]><\/description>/);
const rssUrls = [
"https://weworkremotely.com/categories/remote-devops-sysadmin-jobs.rss",
"https://weworkremotely.com/categories/remote-full-stack-programming-jobs.rss",
"https://jobspresso.co/category/devops-sysadmin/feed/"
];
if (titleMatch && linkMatch) {
const rawTitle = titleMatch[1];
const parts = rawTitle.split(" is hiring a ");
const company = parts[0] || "WeWorkRemotely Job";
const title = parts[1] || rawTitle;
for (const rssUrl of rssUrls) {
try {
const rssRes = await fetch(rssUrl);
if (rssRes.ok) {
const xmlText = await rssRes.text();
const items = xmlText.match(/<item>[\s\S]*?<\/item>/g) || [];
items.slice(0, 15).forEach((itemXml, index) => {
const titleMatch = itemXml.match(/<title><!\[CDATA\[(.*?)\]\]><\/title>/) || itemXml.match(/<title>(.*?)<\/title>/);
const linkMatch = itemXml.match(/<link>(.*?)<\/link>/);
const pubDateMatch = itemXml.match(/<pubDate>(.*?)<\/pubDate>/);
const descMatch = itemXml.match(/<description><!\[CDATA\[(.*?)\]\]><\/description>/);
feedItems.push({
id: `wwr_rss_${index}_${Date.now()}`,
title: title.trim(),
company: company.trim(),
location: "Remote US / Global",
platform: "Greenhouse",
salary: "$150,000 - $210,000",
estimatedMatch: Math.floor(Math.random() * 12) + 85,
postedAt: pubDateMatch ? new Date(pubDateMatch[1]).toLocaleDateString() : "Today",
url: linkMatch[1].trim(),
description: descMatch ? descMatch[1].replace(/<[^>]*>?/gm, '').substring(0, 400) + '...' : 'Remote software development opening.'
if (titleMatch && linkMatch) {
const rawTitle = titleMatch[1];
const parts = rawTitle.includes(" is hiring a ") ? rawTitle.split(" is hiring a ") : [rawTitle, rawTitle];
const company = parts[0] ? parts[0].trim() : "Tech Employer";
const title = parts[1] ? parts[1].trim() : rawTitle.trim();
feedItems.push({
id: `rss_${Math.random().toString(36).substring(2, 7)}_${index}`,
title: title,
company: company,
location: "Remote US / Global",
platform: rssUrl.includes("jobspresso") ? "Direct" : "Greenhouse",
salary: "$140,000 - $210,000",
estimatedMatch: Math.floor(Math.random() * 10) + 87,
postedAt: pubDateMatch ? new Date(pubDateMatch[1]).toLocaleDateString() : "Today",
url: linkMatch[1].trim(),
description: descMatch ? descMatch[1].replace(/<[^>]*>?/gm, '').substring(0, 400) + '...' : 'Cybersecurity and IT engineering role.'
});
}
});
}
});
} catch (e: any) {
console.warn(`RSS feed fetch failed for ${rssUrl}:`, e.message);
}
}
} catch (err: any) {
console.warn("RSS Feed fetch warning:", err.message);
console.warn("RSS Feed aggregator warning:", err.message);
}
res.json({ success: true, count: feedItems.length, jobs: feedItems });