389 lines
13 KiB
JavaScript
389 lines
13 KiB
JavaScript
import mongoose from "mongoose";
|
||
import axios from "axios";
|
||
import dotenv from "dotenv";
|
||
|
||
dotenv.config();
|
||
|
||
/**
|
||
* Import Market Prices Script
|
||
* Downloads all Steam market items and stores them as reference data
|
||
* for quick price lookups when loading inventory or updating prices
|
||
*/
|
||
|
||
const MONGODB_URI =
|
||
process.env.MONGODB_URI || "mongodb://localhost:27017/turbotrades";
|
||
const STEAM_API_KEY =
|
||
process.env.STEAM_APIS_KEY || process.env.STEAM_API_KEY;
|
||
const BASE_URL = "https://api.steamapis.com";
|
||
|
||
// Define market price schema
|
||
const marketPriceSchema = new mongoose.Schema(
|
||
{
|
||
name: {
|
||
type: String,
|
||
required: true,
|
||
index: true,
|
||
},
|
||
game: {
|
||
type: String,
|
||
required: true,
|
||
enum: ["cs2", "rust"],
|
||
index: true,
|
||
},
|
||
appId: {
|
||
type: Number,
|
||
required: true,
|
||
},
|
||
marketHashName: {
|
||
type: String,
|
||
required: true,
|
||
unique: true,
|
||
},
|
||
price: {
|
||
type: Number,
|
||
required: true,
|
||
},
|
||
priceType: {
|
||
type: String,
|
||
enum: ["safe", "median", "mean", "avg", "latest"],
|
||
default: "safe",
|
||
},
|
||
image: {
|
||
type: String,
|
||
default: null,
|
||
},
|
||
borderColor: {
|
||
type: String,
|
||
default: null,
|
||
},
|
||
nameId: {
|
||
type: Number,
|
||
default: null,
|
||
},
|
||
lastUpdated: {
|
||
type: Date,
|
||
default: Date.now,
|
||
},
|
||
},
|
||
{
|
||
timestamps: true,
|
||
collection: "marketprices",
|
||
}
|
||
);
|
||
|
||
// Compound index for fast lookups
|
||
marketPriceSchema.index({ game: 1, name: 1 });
|
||
marketPriceSchema.index({ game: 1, marketHashName: 1 });
|
||
|
||
console.log("\n╔═══════════════════════════════════════════════╗");
|
||
console.log("║ Steam Market Price Import Script ║");
|
||
console.log("╚═══════════════════════════════════════════════╝\n");
|
||
|
||
async function fetchMarketData(game, appId) {
|
||
console.log(`\n📡 Fetching ${game.toUpperCase()} market data...`);
|
||
console.log(` App ID: ${appId}`);
|
||
console.log(` URL: ${BASE_URL}/market/items/${appId}\n`);
|
||
|
||
try {
|
||
const response = await axios.get(`${BASE_URL}/market/items/${appId}`, {
|
||
params: {
|
||
api_key: STEAM_API_KEY,
|
||
},
|
||
timeout: 60000, // 60 second timeout
|
||
});
|
||
|
||
if (!response.data || !response.data.data) {
|
||
console.error(`❌ No data returned for ${game}`);
|
||
return [];
|
||
}
|
||
|
||
const items = response.data.data;
|
||
const itemCount = Object.keys(items).length;
|
||
console.log(`✅ Received ${itemCount} items from API`);
|
||
|
||
// Transform API data to our format
|
||
const marketItems = [];
|
||
|
||
Object.values(items).forEach((item) => {
|
||
// Get the best available price
|
||
const price =
|
||
item.prices?.safe ||
|
||
item.prices?.median ||
|
||
item.prices?.mean ||
|
||
item.prices?.avg ||
|
||
item.prices?.latest;
|
||
|
||
if (!price || price <= 0) {
|
||
return; // Skip items without valid prices
|
||
}
|
||
|
||
const marketHashName = item.market_hash_name || item.market_name;
|
||
const marketName = item.market_name || item.market_hash_name;
|
||
|
||
if (!marketHashName || !marketName) {
|
||
return; // Skip items without names
|
||
}
|
||
|
||
// Determine which price type was used
|
||
let priceType = "safe";
|
||
if (item.prices?.safe) priceType = "safe";
|
||
else if (item.prices?.median) priceType = "median";
|
||
else if (item.prices?.mean) priceType = "mean";
|
||
else if (item.prices?.avg) priceType = "avg";
|
||
else if (item.prices?.latest) priceType = "latest";
|
||
|
||
marketItems.push({
|
||
name: marketName,
|
||
game: game,
|
||
appId: appId,
|
||
marketHashName: marketHashName,
|
||
price: price,
|
||
priceType: priceType,
|
||
image: item.image || null,
|
||
borderColor: item.border_color || null,
|
||
nameId: item.nameID || null,
|
||
lastUpdated: new Date(),
|
||
});
|
||
});
|
||
|
||
console.log(`✅ Processed ${marketItems.length} items with valid prices`);
|
||
return marketItems;
|
||
} catch (error) {
|
||
console.error(`❌ Error fetching ${game} market data:`, error.message);
|
||
|
||
if (error.response?.status === 401) {
|
||
console.error(" 🔑 API key is invalid or expired");
|
||
} else if (error.response?.status === 429) {
|
||
console.error(" ⏱️ Rate limit exceeded");
|
||
} else if (error.response?.status === 403) {
|
||
console.error(" 🚫 Access forbidden - check API subscription");
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async function importToDatabase(MarketPrice, items, game) {
|
||
console.log(`\n💾 Importing ${game.toUpperCase()} items to database...`);
|
||
|
||
let inserted = 0;
|
||
let updated = 0;
|
||
let errors = 0;
|
||
let skipped = 0;
|
||
|
||
// Use bulk operations for better performance
|
||
const bulkOps = [];
|
||
|
||
for (const item of items) {
|
||
bulkOps.push({
|
||
updateOne: {
|
||
filter: { marketHashName: item.marketHashName },
|
||
update: { $set: item },
|
||
upsert: true,
|
||
},
|
||
});
|
||
|
||
// Execute in batches of 1000
|
||
if (bulkOps.length >= 1000) {
|
||
try {
|
||
const result = await MarketPrice.bulkWrite(bulkOps);
|
||
inserted += result.upsertedCount;
|
||
updated += result.modifiedCount;
|
||
console.log(
|
||
` 📦 Batch complete: ${inserted} inserted, ${updated} updated`
|
||
);
|
||
bulkOps.length = 0; // Clear array
|
||
} catch (error) {
|
||
console.error(` ❌ Batch error:`, error.message);
|
||
errors += bulkOps.length;
|
||
bulkOps.length = 0;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Execute remaining items
|
||
if (bulkOps.length > 0) {
|
||
try {
|
||
const result = await MarketPrice.bulkWrite(bulkOps);
|
||
inserted += result.upsertedCount;
|
||
updated += result.modifiedCount;
|
||
} catch (error) {
|
||
console.error(` ❌ Final batch error:`, error.message);
|
||
errors += bulkOps.length;
|
||
}
|
||
}
|
||
|
||
console.log(`\n✅ ${game.toUpperCase()} import complete:`);
|
||
console.log(` 📥 Inserted: ${inserted}`);
|
||
console.log(` 🔄 Updated: ${updated}`);
|
||
if (errors > 0) {
|
||
console.log(` ❌ Errors: ${errors}`);
|
||
}
|
||
if (skipped > 0) {
|
||
console.log(` ⏭️ Skipped: ${skipped}`);
|
||
}
|
||
|
||
return { inserted, updated, errors, skipped };
|
||
}
|
||
|
||
async function main() {
|
||
// Check API key
|
||
if (!STEAM_API_KEY) {
|
||
console.error("❌ ERROR: Steam API key not configured!\n");
|
||
console.error("Please set one of these environment variables:");
|
||
console.error(" - STEAM_APIS_KEY (recommended)");
|
||
console.error(" - STEAM_API_KEY (fallback)\n");
|
||
console.error("Get your API key from: https://steamapis.com/\n");
|
||
process.exit(1);
|
||
}
|
||
|
||
console.log("🔑 API Key: ✓ Configured");
|
||
console.log(` First 10 chars: ${STEAM_API_KEY.substring(0, 10)}...`);
|
||
console.log(`📡 Database: ${MONGODB_URI}\n`);
|
||
|
||
try {
|
||
// Connect to MongoDB
|
||
console.log("🔌 Connecting to MongoDB...");
|
||
await mongoose.connect(MONGODB_URI);
|
||
console.log("✅ Connected to database\n");
|
||
|
||
// Create or get MarketPrice model
|
||
const MarketPrice =
|
||
mongoose.models.MarketPrice ||
|
||
mongoose.model("MarketPrice", marketPriceSchema);
|
||
|
||
console.log("─────────────────────────────────────────────────");
|
||
|
||
// Get current counts
|
||
const cs2Count = await MarketPrice.countDocuments({ game: "cs2" });
|
||
const rustCount = await MarketPrice.countDocuments({ game: "rust" });
|
||
|
||
console.log("\n📊 Current Database Status:");
|
||
console.log(` CS2: ${cs2Count} items`);
|
||
console.log(` Rust: ${rustCount} items`);
|
||
|
||
console.log("\n─────────────────────────────────────────────────");
|
||
|
||
// Fetch and import CS2 items
|
||
console.log("\n🎮 COUNTER-STRIKE 2 (CS2)");
|
||
console.log("─────────────────────────────────────────────────");
|
||
|
||
const cs2Items = await fetchMarketData("cs2", 730);
|
||
const cs2Results = await importToDatabase(MarketPrice, cs2Items, "cs2");
|
||
|
||
console.log("\n─────────────────────────────────────────────────");
|
||
|
||
// Fetch and import Rust items
|
||
console.log("\n🔧 RUST");
|
||
console.log("─────────────────────────────────────────────────");
|
||
|
||
const rustItems = await fetchMarketData("rust", 252490);
|
||
const rustResults = await importToDatabase(MarketPrice, rustItems, "rust");
|
||
|
||
console.log("\n═════════════════════════════════════════════════");
|
||
console.log("\n📊 FINAL SUMMARY\n");
|
||
|
||
console.log("🎮 CS2:");
|
||
console.log(` Total Items: ${cs2Items.length}`);
|
||
console.log(` Inserted: ${cs2Results.inserted}`);
|
||
console.log(` Updated: ${cs2Results.updated}`);
|
||
console.log(` Errors: ${cs2Results.errors}`);
|
||
|
||
console.log("\n🔧 Rust:");
|
||
console.log(` Total Items: ${rustItems.length}`);
|
||
console.log(` Inserted: ${rustResults.inserted}`);
|
||
console.log(` Updated: ${rustResults.updated}`);
|
||
console.log(` Errors: ${rustResults.errors}`);
|
||
|
||
const totalItems = cs2Items.length + rustItems.length;
|
||
const totalInserted = cs2Results.inserted + rustResults.inserted;
|
||
const totalUpdated = cs2Results.updated + rustResults.updated;
|
||
const totalErrors = cs2Results.errors + rustResults.errors;
|
||
|
||
console.log("\n🎉 Grand Total:");
|
||
console.log(` Total Items: ${totalItems}`);
|
||
console.log(` Inserted: ${totalInserted}`);
|
||
console.log(` Updated: ${totalUpdated}`);
|
||
console.log(` Errors: ${totalErrors}`);
|
||
|
||
// Get final counts
|
||
const finalCs2Count = await MarketPrice.countDocuments({ game: "cs2" });
|
||
const finalRustCount = await MarketPrice.countDocuments({ game: "rust" });
|
||
const finalTotal = await MarketPrice.countDocuments();
|
||
|
||
console.log("\n📦 Database Now Contains:");
|
||
console.log(` CS2: ${finalCs2Count} items`);
|
||
console.log(` Rust: ${finalRustCount} items`);
|
||
console.log(` Total: ${finalTotal} items`);
|
||
|
||
console.log("\n─────────────────────────────────────────────────");
|
||
|
||
// Show sample items
|
||
console.log("\n💎 Sample Items (Highest Priced):\n");
|
||
|
||
const sampleItems = await MarketPrice.find()
|
||
.sort({ price: -1 })
|
||
.limit(5)
|
||
.select("name game price priceType");
|
||
|
||
sampleItems.forEach((item, index) => {
|
||
console.log(` ${index + 1}. [${item.game.toUpperCase()}] ${item.name}`);
|
||
console.log(` Price: $${item.price.toFixed(2)} (${item.priceType})`);
|
||
});
|
||
|
||
console.log("\n═════════════════════════════════════════════════");
|
||
console.log("\n✅ Import completed successfully!\n");
|
||
|
||
console.log("💡 Next Steps:");
|
||
console.log(" 1. Use these prices for inventory loading");
|
||
console.log(" 2. Query by: MarketPrice.findOne({ marketHashName: name })");
|
||
console.log(" 3. Update periodically with: node import-market-prices.js\n");
|
||
|
||
console.log("📚 Usage Example:");
|
||
console.log(' const price = await MarketPrice.findOne({ ');
|
||
console.log(' marketHashName: "AK-47 | Redline (Field-Tested)"');
|
||
console.log(" });");
|
||
console.log(" console.log(price.price); // e.g., 12.50\n");
|
||
|
||
// Disconnect
|
||
await mongoose.disconnect();
|
||
console.log("👋 Disconnected from database\n");
|
||
|
||
process.exit(0);
|
||
} catch (error) {
|
||
console.error("\n❌ FATAL ERROR:");
|
||
console.error(` ${error.message}\n`);
|
||
|
||
if (error.message.includes("ECONNREFUSED")) {
|
||
console.error("🔌 MongoDB Connection Failed:");
|
||
console.error(" - Is MongoDB running?");
|
||
console.error(" - Check MONGODB_URI in .env");
|
||
console.error(` - Current URI: ${MONGODB_URI}\n`);
|
||
}
|
||
|
||
console.error("Stack trace:");
|
||
console.error(error.stack);
|
||
console.error();
|
||
|
||
if (mongoose.connection.readyState === 1) {
|
||
await mongoose.disconnect();
|
||
console.log("👋 Disconnected from database\n");
|
||
}
|
||
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
// Handle ctrl+c gracefully
|
||
process.on("SIGINT", async () => {
|
||
console.log("\n\n⚠️ Import interrupted by user");
|
||
if (mongoose.connection.readyState === 1) {
|
||
await mongoose.disconnect();
|
||
console.log("👋 Disconnected from database");
|
||
}
|
||
process.exit(0);
|
||
});
|
||
|
||
// Run the script
|
||
main();
|