first commit
This commit is contained in:
388
import-market-prices.js
Normal file
388
import-market-prices.js
Normal file
@@ -0,0 +1,388 @@
|
||||
import mongoose from "mongoose";
|
||||
import axios from "axios";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
/**
|
||||
* Import Market Prices Script
|
||||
* Downloads all Steam market items and stores them as reference data
|
||||
* for quick price lookups when loading inventory or updating prices
|
||||
*/
|
||||
|
||||
const MONGODB_URI =
|
||||
process.env.MONGODB_URI || "mongodb://localhost:27017/turbotrades";
|
||||
const STEAM_API_KEY =
|
||||
process.env.STEAM_APIS_KEY || process.env.STEAM_API_KEY;
|
||||
const BASE_URL = "https://api.steamapis.com";
|
||||
|
||||
// Define market price schema
|
||||
const marketPriceSchema = new mongoose.Schema(
|
||||
{
|
||||
name: {
|
||||
type: String,
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
game: {
|
||||
type: String,
|
||||
required: true,
|
||||
enum: ["cs2", "rust"],
|
||||
index: true,
|
||||
},
|
||||
appId: {
|
||||
type: Number,
|
||||
required: true,
|
||||
},
|
||||
marketHashName: {
|
||||
type: String,
|
||||
required: true,
|
||||
unique: true,
|
||||
},
|
||||
price: {
|
||||
type: Number,
|
||||
required: true,
|
||||
},
|
||||
priceType: {
|
||||
type: String,
|
||||
enum: ["safe", "median", "mean", "avg", "latest"],
|
||||
default: "safe",
|
||||
},
|
||||
image: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
borderColor: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
nameId: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
lastUpdated: {
|
||||
type: Date,
|
||||
default: Date.now,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
collection: "marketprices",
|
||||
}
|
||||
);
|
||||
|
||||
// Compound index for fast lookups
|
||||
marketPriceSchema.index({ game: 1, name: 1 });
|
||||
marketPriceSchema.index({ game: 1, marketHashName: 1 });
|
||||
|
||||
console.log("\n╔═══════════════════════════════════════════════╗");
|
||||
console.log("║ Steam Market Price Import Script ║");
|
||||
console.log("╚═══════════════════════════════════════════════╝\n");
|
||||
|
||||
async function fetchMarketData(game, appId) {
|
||||
console.log(`\n📡 Fetching ${game.toUpperCase()} market data...`);
|
||||
console.log(` App ID: ${appId}`);
|
||||
console.log(` URL: ${BASE_URL}/market/items/${appId}\n`);
|
||||
|
||||
try {
|
||||
const response = await axios.get(`${BASE_URL}/market/items/${appId}`, {
|
||||
params: {
|
||||
api_key: STEAM_API_KEY,
|
||||
},
|
||||
timeout: 60000, // 60 second timeout
|
||||
});
|
||||
|
||||
if (!response.data || !response.data.data) {
|
||||
console.error(`❌ No data returned for ${game}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const items = response.data.data;
|
||||
const itemCount = Object.keys(items).length;
|
||||
console.log(`✅ Received ${itemCount} items from API`);
|
||||
|
||||
// Transform API data to our format
|
||||
const marketItems = [];
|
||||
|
||||
Object.values(items).forEach((item) => {
|
||||
// Get the best available price
|
||||
const price =
|
||||
item.prices?.safe ||
|
||||
item.prices?.median ||
|
||||
item.prices?.mean ||
|
||||
item.prices?.avg ||
|
||||
item.prices?.latest;
|
||||
|
||||
if (!price || price <= 0) {
|
||||
return; // Skip items without valid prices
|
||||
}
|
||||
|
||||
const marketHashName = item.market_hash_name || item.market_name;
|
||||
const marketName = item.market_name || item.market_hash_name;
|
||||
|
||||
if (!marketHashName || !marketName) {
|
||||
return; // Skip items without names
|
||||
}
|
||||
|
||||
// Determine which price type was used
|
||||
let priceType = "safe";
|
||||
if (item.prices?.safe) priceType = "safe";
|
||||
else if (item.prices?.median) priceType = "median";
|
||||
else if (item.prices?.mean) priceType = "mean";
|
||||
else if (item.prices?.avg) priceType = "avg";
|
||||
else if (item.prices?.latest) priceType = "latest";
|
||||
|
||||
marketItems.push({
|
||||
name: marketName,
|
||||
game: game,
|
||||
appId: appId,
|
||||
marketHashName: marketHashName,
|
||||
price: price,
|
||||
priceType: priceType,
|
||||
image: item.image || null,
|
||||
borderColor: item.border_color || null,
|
||||
nameId: item.nameID || null,
|
||||
lastUpdated: new Date(),
|
||||
});
|
||||
});
|
||||
|
||||
console.log(`✅ Processed ${marketItems.length} items with valid prices`);
|
||||
return marketItems;
|
||||
} catch (error) {
|
||||
console.error(`❌ Error fetching ${game} market data:`, error.message);
|
||||
|
||||
if (error.response?.status === 401) {
|
||||
console.error(" 🔑 API key is invalid or expired");
|
||||
} else if (error.response?.status === 429) {
|
||||
console.error(" ⏱️ Rate limit exceeded");
|
||||
} else if (error.response?.status === 403) {
|
||||
console.error(" 🚫 Access forbidden - check API subscription");
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function importToDatabase(MarketPrice, items, game) {
|
||||
console.log(`\n💾 Importing ${game.toUpperCase()} items to database...`);
|
||||
|
||||
let inserted = 0;
|
||||
let updated = 0;
|
||||
let errors = 0;
|
||||
let skipped = 0;
|
||||
|
||||
// Use bulk operations for better performance
|
||||
const bulkOps = [];
|
||||
|
||||
for (const item of items) {
|
||||
bulkOps.push({
|
||||
updateOne: {
|
||||
filter: { marketHashName: item.marketHashName },
|
||||
update: { $set: item },
|
||||
upsert: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Execute in batches of 1000
|
||||
if (bulkOps.length >= 1000) {
|
||||
try {
|
||||
const result = await MarketPrice.bulkWrite(bulkOps);
|
||||
inserted += result.upsertedCount;
|
||||
updated += result.modifiedCount;
|
||||
console.log(
|
||||
` 📦 Batch complete: ${inserted} inserted, ${updated} updated`
|
||||
);
|
||||
bulkOps.length = 0; // Clear array
|
||||
} catch (error) {
|
||||
console.error(` ❌ Batch error:`, error.message);
|
||||
errors += bulkOps.length;
|
||||
bulkOps.length = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Execute remaining items
|
||||
if (bulkOps.length > 0) {
|
||||
try {
|
||||
const result = await MarketPrice.bulkWrite(bulkOps);
|
||||
inserted += result.upsertedCount;
|
||||
updated += result.modifiedCount;
|
||||
} catch (error) {
|
||||
console.error(` ❌ Final batch error:`, error.message);
|
||||
errors += bulkOps.length;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n✅ ${game.toUpperCase()} import complete:`);
|
||||
console.log(` 📥 Inserted: ${inserted}`);
|
||||
console.log(` 🔄 Updated: ${updated}`);
|
||||
if (errors > 0) {
|
||||
console.log(` ❌ Errors: ${errors}`);
|
||||
}
|
||||
if (skipped > 0) {
|
||||
console.log(` ⏭️ Skipped: ${skipped}`);
|
||||
}
|
||||
|
||||
return { inserted, updated, errors, skipped };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// Check API key
|
||||
if (!STEAM_API_KEY) {
|
||||
console.error("❌ ERROR: Steam API key not configured!\n");
|
||||
console.error("Please set one of these environment variables:");
|
||||
console.error(" - STEAM_APIS_KEY (recommended)");
|
||||
console.error(" - STEAM_API_KEY (fallback)\n");
|
||||
console.error("Get your API key from: https://steamapis.com/\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("🔑 API Key: ✓ Configured");
|
||||
console.log(` First 10 chars: ${STEAM_API_KEY.substring(0, 10)}...`);
|
||||
console.log(`📡 Database: ${MONGODB_URI}\n`);
|
||||
|
||||
try {
|
||||
// Connect to MongoDB
|
||||
console.log("🔌 Connecting to MongoDB...");
|
||||
await mongoose.connect(MONGODB_URI);
|
||||
console.log("✅ Connected to database\n");
|
||||
|
||||
// Create or get MarketPrice model
|
||||
const MarketPrice =
|
||||
mongoose.models.MarketPrice ||
|
||||
mongoose.model("MarketPrice", marketPriceSchema);
|
||||
|
||||
console.log("─────────────────────────────────────────────────");
|
||||
|
||||
// Get current counts
|
||||
const cs2Count = await MarketPrice.countDocuments({ game: "cs2" });
|
||||
const rustCount = await MarketPrice.countDocuments({ game: "rust" });
|
||||
|
||||
console.log("\n📊 Current Database Status:");
|
||||
console.log(` CS2: ${cs2Count} items`);
|
||||
console.log(` Rust: ${rustCount} items`);
|
||||
|
||||
console.log("\n─────────────────────────────────────────────────");
|
||||
|
||||
// Fetch and import CS2 items
|
||||
console.log("\n🎮 COUNTER-STRIKE 2 (CS2)");
|
||||
console.log("─────────────────────────────────────────────────");
|
||||
|
||||
const cs2Items = await fetchMarketData("cs2", 730);
|
||||
const cs2Results = await importToDatabase(MarketPrice, cs2Items, "cs2");
|
||||
|
||||
console.log("\n─────────────────────────────────────────────────");
|
||||
|
||||
// Fetch and import Rust items
|
||||
console.log("\n🔧 RUST");
|
||||
console.log("─────────────────────────────────────────────────");
|
||||
|
||||
const rustItems = await fetchMarketData("rust", 252490);
|
||||
const rustResults = await importToDatabase(MarketPrice, rustItems, "rust");
|
||||
|
||||
console.log("\n═════════════════════════════════════════════════");
|
||||
console.log("\n📊 FINAL SUMMARY\n");
|
||||
|
||||
console.log("🎮 CS2:");
|
||||
console.log(` Total Items: ${cs2Items.length}`);
|
||||
console.log(` Inserted: ${cs2Results.inserted}`);
|
||||
console.log(` Updated: ${cs2Results.updated}`);
|
||||
console.log(` Errors: ${cs2Results.errors}`);
|
||||
|
||||
console.log("\n🔧 Rust:");
|
||||
console.log(` Total Items: ${rustItems.length}`);
|
||||
console.log(` Inserted: ${rustResults.inserted}`);
|
||||
console.log(` Updated: ${rustResults.updated}`);
|
||||
console.log(` Errors: ${rustResults.errors}`);
|
||||
|
||||
const totalItems = cs2Items.length + rustItems.length;
|
||||
const totalInserted = cs2Results.inserted + rustResults.inserted;
|
||||
const totalUpdated = cs2Results.updated + rustResults.updated;
|
||||
const totalErrors = cs2Results.errors + rustResults.errors;
|
||||
|
||||
console.log("\n🎉 Grand Total:");
|
||||
console.log(` Total Items: ${totalItems}`);
|
||||
console.log(` Inserted: ${totalInserted}`);
|
||||
console.log(` Updated: ${totalUpdated}`);
|
||||
console.log(` Errors: ${totalErrors}`);
|
||||
|
||||
// Get final counts
|
||||
const finalCs2Count = await MarketPrice.countDocuments({ game: "cs2" });
|
||||
const finalRustCount = await MarketPrice.countDocuments({ game: "rust" });
|
||||
const finalTotal = await MarketPrice.countDocuments();
|
||||
|
||||
console.log("\n📦 Database Now Contains:");
|
||||
console.log(` CS2: ${finalCs2Count} items`);
|
||||
console.log(` Rust: ${finalRustCount} items`);
|
||||
console.log(` Total: ${finalTotal} items`);
|
||||
|
||||
console.log("\n─────────────────────────────────────────────────");
|
||||
|
||||
// Show sample items
|
||||
console.log("\n💎 Sample Items (Highest Priced):\n");
|
||||
|
||||
const sampleItems = await MarketPrice.find()
|
||||
.sort({ price: -1 })
|
||||
.limit(5)
|
||||
.select("name game price priceType");
|
||||
|
||||
sampleItems.forEach((item, index) => {
|
||||
console.log(` ${index + 1}. [${item.game.toUpperCase()}] ${item.name}`);
|
||||
console.log(` Price: $${item.price.toFixed(2)} (${item.priceType})`);
|
||||
});
|
||||
|
||||
console.log("\n═════════════════════════════════════════════════");
|
||||
console.log("\n✅ Import completed successfully!\n");
|
||||
|
||||
console.log("💡 Next Steps:");
|
||||
console.log(" 1. Use these prices for inventory loading");
|
||||
console.log(" 2. Query by: MarketPrice.findOne({ marketHashName: name })");
|
||||
console.log(" 3. Update periodically with: node import-market-prices.js\n");
|
||||
|
||||
console.log("📚 Usage Example:");
|
||||
console.log(' const price = await MarketPrice.findOne({ ');
|
||||
console.log(' marketHashName: "AK-47 | Redline (Field-Tested)"');
|
||||
console.log(" });");
|
||||
console.log(" console.log(price.price); // e.g., 12.50\n");
|
||||
|
||||
// Disconnect
|
||||
await mongoose.disconnect();
|
||||
console.log("👋 Disconnected from database\n");
|
||||
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error("\n❌ FATAL ERROR:");
|
||||
console.error(` ${error.message}\n`);
|
||||
|
||||
if (error.message.includes("ECONNREFUSED")) {
|
||||
console.error("🔌 MongoDB Connection Failed:");
|
||||
console.error(" - Is MongoDB running?");
|
||||
console.error(" - Check MONGODB_URI in .env");
|
||||
console.error(` - Current URI: ${MONGODB_URI}\n`);
|
||||
}
|
||||
|
||||
console.error("Stack trace:");
|
||||
console.error(error.stack);
|
||||
console.error();
|
||||
|
||||
if (mongoose.connection.readyState === 1) {
|
||||
await mongoose.disconnect();
|
||||
console.log("👋 Disconnected from database\n");
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle ctrl+c gracefully
|
||||
process.on("SIGINT", async () => {
|
||||
console.log("\n\n⚠️ Import interrupted by user");
|
||||
if (mongoose.connection.readyState === 1) {
|
||||
await mongoose.disconnect();
|
||||
console.log("👋 Disconnected from database");
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
// Run the script
|
||||
main();
|
||||
Reference in New Issue
Block a user