From f4e6a714a4e28ec8add6fb5ae3baa6b91ad8ef10 Mon Sep 17 00:00:00 2001 From: Mr_Goldberg Date: Sun, 14 Aug 2022 14:54:12 -0400 Subject: [PATCH] generate_emu_config can now generate items config. Remove the old generate_game_infos.cpp script. --- generate_game_infos/generate_game_infos.cpp | 591 -------------------- scripts/generate_emu_config.py | 57 +- 2 files changed, 53 insertions(+), 595 deletions(-) delete mode 100644 generate_game_infos/generate_game_infos.cpp diff --git a/generate_game_infos/generate_game_infos.cpp b/generate_game_infos/generate_game_infos.cpp deleted file mode 100644 index 39a4447..0000000 --- a/generate_game_infos/generate_game_infos.cpp +++ /dev/null @@ -1,591 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -class CurlGlobal -{ - bool _init; - - CurlGlobal() :_init(false) {} - - ~CurlGlobal() { cleanup(); } - -public: - static CurlGlobal& Inst() - { - static CurlGlobal _this; - return _this; - } - - CURLcode init(long flags = CURL_GLOBAL_DEFAULT) { return curl_global_init(flags); } - void cleanup() - { - if (_init) - { - curl_global_cleanup(); - _init = false; - } - } -}; - -class CurlEasy -{ - CURL* _me; - bool _init; - std::string _buffer; - - static int writer(char* data, size_t size, size_t nmemb, - CurlEasy *_this) - { - if (_this == nullptr) - return 0; - - _this->_buffer.append(data, size * nmemb); - - return size * nmemb; - } - -public: - CurlEasy() :_me(nullptr), _init(false) {} - ~CurlEasy() { cleanup(); } - - bool init() - { - _init = (_me = curl_easy_init()) != nullptr; - if (_init) - { - if (curl_easy_setopt(_me, CURLOPT_WRITEFUNCTION, writer) != CURLE_OK) - { - cleanup(); - return false; - } - - if (curl_easy_setopt(_me, CURLOPT_WRITEDATA, this) != CURLE_OK) - { - cleanup(); - return false; - } - } - return _init; - } - - void cleanup() - { - if (_init) - { - curl_easy_cleanup(_me); - } - } - - CURLcode set_url(const std::string& url) - { - return curl_easy_setopt(_me, CURLOPT_URL, url.c_str()); - } - - CURLcode skip_verifypeer(bool skip = true) - { - return curl_easy_setopt(_me, CURLOPT_SSL_VERIFYPEER, skip ? 0L : 1L); - } - - CURLcode skip_verifhost(bool skip = true) - { - return curl_easy_setopt(_me, CURLOPT_SSL_VERIFYHOST, skip ? 0L : 1L); - } - - CURLcode connect_only(bool connect = true) - { - return curl_easy_setopt(_me, CURLOPT_CONNECT_ONLY, connect ? 1L : 0L); - } - - CURLcode perform() - { - _buffer.clear(); - return curl_easy_perform(_me); - } - - CURLcode recv(void *buffer, size_t buflen, size_t* read_len) - { - return curl_easy_recv(_me, buffer, buflen, read_len); - } - - CURLcode get_html_code(long &code) - { - return curl_easy_getinfo(_me, CURLINFO_RESPONSE_CODE, &code); - } - - std::string const& get_answer() const { return _buffer; } -}; - -// Get all steam appid with their name: http://api.steampowered.com/ISteamApps/GetAppList/v2/ -// Steam storefront webapi: https://wiki.teamfortress.com/wiki/User:RJackson/StorefrontAPI -// http://api.steampowered.com/ISteamUserStats/GetSchemaForGame/v2/?key=&appid= -/* -{ - "game" : { - "gameName" : "", - "availableGameStats" : { - "achievements" : { - ("" : { - "name" : "achievement_name", - "displayName" : "achievement name on screen", - "hidden" : (0|1), - ["description" : "",] - "icon" : "", - "icongray" : "" - }, - ...) - } - } - } -} -*/ -// Get appid infos: http://store.steampowered.com/api/appdetails/?appids=218620 -/* -"appid" : { - "success" : (true|false), - (success == true "data" : { - ... - "name" : "", - "steam_appid" : , - (OPT "dlc" : [, ]), - "header_image" : "" <-- Use this in the overlay ? - (OPT "achievements" : { - "total" : - }), - "background" : "" <-- Use this as the overlay background ? - (OPT "packages" : [, ]) - }) -} -*/ -// --------------------------------- -// -- Special thanks to psychonic -- -// --------------------------------- -// Get game items definition digest (Phase1): https://api.steampowered.com/IInventoryService/GetItemDefMeta/v1?key=&appid=218620 -/* -{ - "response": { - "modified": 1566848385, - "digest": "3CDFC1CC1AC2B0D55D12C1C130F4294BDD6DF8D0" - } -} -*/ - -// Get game items definition: https://api.steampowered.com/IGameInventory/GetItemDefArchive/v0001?appid=218620&digest= -/* -[ - { - "appid":"218620", - "itemdefid":"0", - "Timestamp":"2016-04-08T18:00:21.3643085Z", - "modified":"20160408T180021Z", - "date_created":"20160408T180021Z", - "type":"", - "display_type":"", - "name":"", - "quantity":0, - "description":"", - "tradable":false, - "marketable":false, - "commodity":false, - "drop_interval":0, - "drop_max_per_window":0, - "workshopid":"0" - }, - { - "appid":"218620", - "itemdefid":"50002", - "Timestamp":"2015-11-13T16:01:18.0338618Z", - "modified":"20151113T160117Z", - "date_created":"20151113T160117Z", - "type":"item", - "display_type":"", - "name":"Sputnik Safe", - "quantity":0, - "description":"[color=#2360D8]THE JUDGE SHOTGUN | Pixel [/color]\n[color=#2360D8]KOBUS 90 SUBMACHINE GUN | Red Stars[/color]\n[color=#2360D8]PLAINSRIDER BOW | Arctic Plains[/color]\n[color=#2360D8]GRUBER KURZ PISTOL | Little Leopard[/color]\n[color=#2360D8]HRL-7 ROCKET LAUNCHER | Headline[/color]\n[color=#2360D8]LOCOMOTIVE 12G SHOTGUN | Cosmonaut[/color]\n[color=#9900FF]FLAMETHROWER | St. Basil[/color]\n[color=#9900FF]JP36 RIFLE | Ice Leopard [/color]\n[color=#9900FF]CAR-4 RIFLE | Stripe On[/color]\n[color=#9900FF]BRONCO .44 REVOLVER | Black Bull[/color]\n[color=#FF00FF]BERNETTI 9 PISTOL | Angry Bear[/color]\n[color=#FF00FF]THANATOS .50 CAL SNIPER RIFLE | Matrjoschka[/color]\n[color=#FF00FF]M308 RIFLE | Helmet Space Program[/color]\n[color=#FF0000]CLARION RIFLE | Breaching Owl[/color]\n[color=#FF0000]MOSCONI 12G SHOTGUN | Bullet Bear Gun[/color]\n[color=#FFAA00]or an exceedingly rare special item![/color]", - "icon_url":"http://media.overkillsoftware.com/economy42gF2Y/safes_weapon_01.png", - "icon_url_large":"http://media.overkillsoftware.com/economy42gF2Y/safes_weapon_01.png", - "store_tags":"safe;sputnik safe;", - "tradable":true, - "marketable":true, - "commodity":false, - "drop_interval":0, - "drop_max_per_window":0, - "workshopid":"0", - "dsl_bonus":"false", - "item_name":"weapon_01", - "item_slot":"safes" - } -*/ - -#ifdef max -#undef max -#endif - -std::string steam_apikey; -std::string app_id; -std::string output_path; - -#if defined(WIN32) || defined(_WIN32) -#include - -static bool create_directory(std::string const& strPath) -{ - DWORD dwAttrib = GetFileAttributesA(strPath.c_str()); - - if (dwAttrib != INVALID_FILE_ATTRIBUTES && dwAttrib & FILE_ATTRIBUTE_DIRECTORY) - return true; - - return CreateDirectoryA(strPath.c_str(), NULL); -} -#elif defined(__linux__) -#include -#include -#include - -static bool create_directory(std::string const& strPath) -{ - struct stat sb; - - if (stat(strPath.c_str(), &sb) != 0) - { - return mkdir(strPath.c_str(), 0755) == 0; - } - if (S_ISDIR(sb.st_mode)) - return true; - - return false; -} - -#endif - -static void generate_achievements(CurlEasy &easy) -{ - std::string url = "https://api.steampowered.com/ISteamUserStats/GetSchemaForGame/v2/?key="; - url += steam_apikey; - url += "&appid="; - url += app_id; - easy.set_url(url); - easy.perform(); - try - { - std::ofstream ach_file(output_path + "/achievements.json", std::ios::trunc | std::ios::out); - nlohmann::json json = nlohmann::json::parse(easy.get_answer()); - nlohmann::json output_json = nlohmann::json::array(); - - bool first = true; - int i = 0; - for (auto& item : json["game"]["availableGameStats"]["achievements"].items()) - { - output_json[i]["name"] = item.value()["name"]; - output_json[i]["displayName"] = item.value()["displayName"]; - output_json[i]["hidden"] = std::to_string(item.value()["hidden"].get()); - try - { - if( !item.value()["description"].is_null() ) - output_json[i]["description"] = item.value()["description"]; - else - output_json[i]["description"] = ""; - } - catch (...) - { - output_json[i]["description"] = ""; - } - - { - std::string icon_path = "images/" + item.value()["name"].get() + ".jpg"; - std::ofstream achievement_icon(output_path + "/" + icon_path, std::ios::out | std::ios::trunc | std::ios::binary); - if (!achievement_icon) - { - std::cerr << "Cannot create achievement icon \"" << icon_path << "\"" << std::endl; - return; - } - easy.set_url(item.value()["icon"]); - easy.perform(); - - std::string picture = easy.get_answer(); - achievement_icon.write(picture.c_str(), picture.length()); - - output_json[i]["icon"] = icon_path; - - } - { - std::string icon_path = "images/" + item.value()["name"].get() + "_gray.jpg"; - std::ofstream achievement_icon(output_path + "/" + icon_path, std::ios::out | std::ios::trunc | std::ios::binary); - if (!achievement_icon) - { - std::cerr << "Cannot create achievement icon \"" << icon_path << "\"" << std::endl; - return; - } - easy.set_url(item.value()["icongray"]); - easy.perform(); - - std::string picture = easy.get_answer(); - achievement_icon.write(picture.c_str(), picture.length()); - - output_json[i]["icongray"] = icon_path; - } - ++i; - } - ach_file << std::setw(2) << output_json; - } - catch (std::exception& e) - { - std::cerr << "Failed to get infos: "; - long code; - if (easy.get_html_code(code) == CURLE_OK && code == 403) - { - std::cerr << "Error in webapi key"; - } - else - { - std::cerr << "Error while parsing json. Try to go at " << url << " and see what you can do to build your achivements.json"; - } - std::cerr << std::endl; - } -} - -template -using my_workaround_fifo_map = nlohmann::fifo_map, A>; -using fifo_json = nlohmann::basic_json; - - -static void generate_items(CurlEasy& easy) -{ - std::string url = "https://api.steampowered.com/IInventoryService/GetItemDefMeta/v1?key="; - url += steam_apikey; - url += "&appid="; - url += app_id; - - easy.set_url(url); - easy.perform(); - - try - { - nlohmann::json json = nlohmann::json::parse(easy.get_answer()); - std::string digest = json["response"]["digest"]; - - url = "https://api.steampowered.com/IGameInventory/GetItemDefArchive/v0001?appid="; - url += app_id; - url += "&digest="; - url += digest; - - easy.set_url(url); - easy.perform(); - - fifo_json item_json; - fifo_json default_item_json; - - json = nlohmann::json::parse(easy.get_answer()); - std::ofstream items_file(output_path + "/items.json", std::ios::trunc | std::ios::out); - std::ofstream default_items_file(output_path + "/default_items.json", std::ios::trunc | std::ios::out); - - for (auto &i : json) - { - for (auto j = i.begin(); j != i.end(); ++j) - { - //if (j.key() == "itemdefid") - //{ - // j.value() = std::stoll(j.value().get()); - //} - //else - { - nlohmann::json& v = j.value(); - switch (v.type()) - { - case nlohmann::json::value_t::boolean: - v = (v.get() ? "true" : "false"); - break; - - case nlohmann::json::value_t::number_float: - v = std::to_string(v.get()); - break; - - case nlohmann::json::value_t::number_integer: - v = std::to_string(v.get()); - break; - - case nlohmann::json::value_t::number_unsigned: - v = std::to_string(v.get()); - break; - } - } - } - item_json[i["itemdefid"].get()] = i; - default_item_json[i["itemdefid"].get()] = 1; - } - - items_file << std::setw(2) << item_json; - default_items_file << std::setw(2) << default_item_json; - } - catch (std::exception& e) - { - std::cerr << "Failed to get infos: "; - long code; - if (easy.get_html_code(code) == CURLE_OK && code == 403) - { - std::cerr << "Error in webapi key"; - } - else - { - std::cerr << "Error while parsing json. Try to go at " << url << " and see what you can do to build your items.json"; - } - std::cerr << std::endl; - } -} - -static std::string get_appid_name(CurlEasy& easy, uint32_t appid) -{ - static std::map appid_names; - static bool done; - - if (!done) { - std::string url = "https://api.steampowered.com/ISteamApps/GetAppList/v2/"; - std::cout << "getting app list" << std::endl; - easy.set_url(url); - easy.perform(); - try - { - nlohmann::json json = nlohmann::json::parse(easy.get_answer()); - for (auto &app : json["applist"]["apps"]) { - appid_names[app["appid"].get()] = app["name"].get(); - } - - done = true; - } - catch (std::exception& e) - { - std::cerr << "Failed to get infos: "; - long code; - if (easy.get_html_code(code) == CURLE_OK && code == 403) - { - std::cerr << "Error 403 while getting app list"; - } - else - { - std::cerr << "Error while parsing json. With " << url << ""; - } - std::cerr << std::endl; - } - } - - if (done) { - if (!appid_names.count(appid)) { - std::cout << "getting app name: " << appid << std::endl; - std::string s_appid = std::to_string(appid); - std::string url = "https://store.steampowered.com/api/appdetails/?appids=" + s_appid; - easy.set_url(url); - easy.perform(); - nlohmann::json json = nlohmann::json::parse(easy.get_answer()); - appid_names[appid] = json[s_appid]["data"]["name"].get(); - } - - return appid_names[appid]; - } - - return ""; -} - -static void generate_dlcs(CurlEasy& easy) -{ - std::string base_url = "https://store.steampowered.com/api/appdetails/?appids="; - std::string url = base_url + app_id; - easy.set_url(url); - easy.perform(); - - try - { - nlohmann::json json = nlohmann::json::parse(easy.get_answer()); - std::list dlcs; - std::map dlc_names; - - for (auto& dlc : json[app_id]["data"]["dlc"]) - { - dlcs.push_back(dlc.get()); - } - - std::ofstream dlc_file(output_path + "/DLC.txt", std::ios::trunc | std::ios::out); - for (auto &dlc: dlcs) { - dlc_file << dlc << "=" << get_appid_name(easy, dlc) << std::endl; - } - } - catch (std::exception& e) - { - std::cerr << "Failed to get infos: "; - long code; - if (easy.get_html_code(code) == CURLE_OK && code == 403) - { - std::cerr << "Error 403 while getting dlcs"; - } - else - { - std::cerr << "Error while parsing json. With " << url << ""; - } - std::cerr << std::endl; - } -} - -int main(int argc, char **argv) -{ - CurlGlobal& cglobal = CurlGlobal::Inst(); - cglobal.init(); - - CurlEasy easy; - if (easy.init()) - { - easy.skip_verifypeer(); - - if (argc > 2) { - app_id = argv[2]; - steam_apikey = argv[1]; - } else { - std::cout << "Usage: " << argv[0] << " steam_api_key app_id " << std::endl; - std::cout << "Enter the game appid: "; - std::cin >> app_id; - std::cout << "Enter your webapi key: "; - std::cin.clear(); - std::cin.ignore(std::numeric_limits::max(), '\n'); - std::cin >> steam_apikey; - } - - if (argc > 3) { - output_path = argv[3]; - } else { - output_path = app_id; - create_directory(output_path); - output_path += "/steam_settings"; - } - - if (!create_directory(output_path)) - { - std::cerr << "Cannot create directory: " << output_path << std::endl; - return -1; - } - - if (!create_directory(output_path + "/images")) - { - std::cerr << "Cannot create directory \"images\"" << std::endl; - return -1; - } - - { - std::ofstream appid_file(output_path + "/steam_appid.txt", std::ios::trunc | std::ios::out); - appid_file << app_id; - } - - std::cout << "Generating DLC.txt" << std::endl; - generate_dlcs(easy); - std::cout << "Generating achievements" << std::endl; - generate_achievements(easy); - std::cout << "Generating items" << std::endl; - generate_items(easy); - } -} diff --git a/scripts/generate_emu_config.py b/scripts/generate_emu_config.py index fac92f5..31a8dbd 100644 --- a/scripts/generate_emu_config.py +++ b/scripts/generate_emu_config.py @@ -112,7 +112,7 @@ def download_achievement_images(game_id, image_names, output_folder): print("error could not download", name) q.task_done() - num_threads = 5 + num_threads = 20 for i in range(num_threads): threading.Thread(target=downloader_thread, daemon=True).start() @@ -143,7 +143,8 @@ def generate_achievement_stats(client, game_id, output_directory, backup_directo images_to_download.append(ach["icon_gray"]) break else: - print("no schema", out) + pass + # print("no schema", out) if (len(images_to_download) > 0): if not os.path.exists(achievement_images_dir): @@ -189,6 +190,26 @@ def download_published_file(client, published_file_id, backup_directory): return data return None +def get_inventory_info(client, game_id): + return client.send_um_and_wait('Inventory.GetItemDefMeta#1', { + 'appid': game_id + }) + +def generate_inventory(client, game_id): + inventory = get_inventory_info(client, appid) + if inventory.header.eresult != EResult.OK: + return None + + url = "https://api.steampowered.com/IGameInventory/GetItemDefArchive/v0001?appid={}&digest={}".format(game_id, inventory.body.digest) + try: + with urllib.request.urlopen(url) as response: + return response.read() + except urllib.error.HTTPError as e: + print("HTTPError getting", url, e.code) + except urllib.error.URLError as e: + print("URLError getting", url, e.code) + return None + def get_dlc(raw_infos): try: try: @@ -226,8 +247,8 @@ for appid in appids: if "common" in game_info: game_info_common = game_info["common"] - if "community_visible_stats" in game_info_common: - generate_achievement_stats(client, appid, out_dir, backup_dir) + #if "community_visible_stats" in game_info_common: #NOTE: checking this seems to skip stats on a few games so it's commented out + generate_achievement_stats(client, appid, out_dir, backup_dir) if "supported_languages" in game_info_common: with open(os.path.join(out_dir, "supported_languages.txt"), 'w') as f: languages = game_info_common["supported_languages"] @@ -295,6 +316,34 @@ for appid in appids: print(id, controller_type) out_vdf = download_published_file(client, int(id), os.path.join(backup_dir, controller_type + str(id))) + inventory_data = generate_inventory(client, appid) + if inventory_data is not None: + out_inventory = {} + default_items = {} + inventory = json.loads(inventory_data.rstrip(b"\x00")) + raw_inventory = json.dumps(inventory, indent=4) + with open(os.path.join(backup_dir, "inventory.json"), "w") as f: + f.write(raw_inventory) + for i in inventory: + index = str(i["itemdefid"]) + x = {} + for t in i: + if i[t] is True: + x[t] = "true" + elif i[t] is False: + x[t] = "false" + else: + x[t] = str(i[t]) + out_inventory[index] = x + default_items[index] = 1 + + out_json_inventory = json.dumps(out_inventory, indent=2) + with open(os.path.join(out_dir, "items.json"), "w") as f: + f.write(out_json_inventory) + out_json_inventory = json.dumps(default_items, indent=2) + with open(os.path.join(out_dir, "default_items.json"), "w") as f: + f.write(out_json_inventory) + game_info_backup = json.dumps(game_info, indent=4) with open(os.path.join(backup_dir, "product_info.json"), "w") as f: f.write(game_info_backup)