Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#pragma once
#include <DD4hep/DetFactoryHelper.h>
#include <DD4hep/Primitives.h>
#include <DD4hep/Factories.h>
#include <DD4hep/Printout.h>
#include <fmt/core.h>
#include <filesystem>
#include <iostream>
#include <cstdlib>
#include <string>
namespace fs = std::filesystem;
using namespace dd4hep;
// Function to download files
inline void
EnsureFileFromURLExists(
std::string url,
std::string file,
std::string cache = "",
std::string cmd = "curl --retry 5 -f {0} -o {1}"
) {
// parse cache for environment variables
auto pos = std::string::npos;
while ((pos = cache.find('$')) != std::string::npos) {
auto after = cache.find_first_not_of(
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789"
"_",
pos + 1);
if (after == std::string::npos) after = cache.size(); // cache ends on env var
const std::string env_name(cache.substr(pos + 1, after - pos - 1));
auto env_ptr = std::getenv(env_name.c_str());
const std::string env_value(env_ptr != nullptr ? env_ptr : "");
cache.erase(pos, after - pos);
cache.insert(pos, env_value);
printout(INFO, "FileLoader", "$" + env_name + " -> " + env_value);
}
// create file path
fs::path file_path(file);
// create hash from url, hex of unsigned long long
std::string hash = fmt::format("{:016x}", dd4hep::detail::hash64(url)); // TODO: Use c++20 std::fmt
// create file parent path, if not exists
fs::path parent_path = file_path.parent_path();
if (!fs::exists(parent_path)) {
if (fs::create_directories(parent_path) == false) {
printout(ERROR, "FileLoader", "parent path " + parent_path.string() + " cannot be created");
printout(ERROR, "FileLoader", "check permissions and retry");
std::quick_exit(1);
}
}
// if file exists and is symlink to correct hash
fs::path hash_path(parent_path / hash);
if (fs::exists(file_path)
&& fs::equivalent(file_path, hash_path)) {
printout(INFO, "FileLoader", "Link " + file + " -> hash " + hash + " already exists");
return;
}
// if hash does not exist, we try to retrieve file from cache
if (!fs::exists(hash_path)) {
// recursive loop into cache directory
fs::path cache_path(cache);
printout(INFO, "FileLoader", "Cache " + cache_path.string());
if (fs::exists(cache_path)) {
for (auto const& dir_entry: fs::recursive_directory_iterator(cache_path)) {
if (!dir_entry.is_directory()) continue;
fs::path cache_dir_path = cache_path / dir_entry;
printout(INFO, "FileLoader", "Checking " + cache_dir_path.string());
fs::path cache_hash_path = cache_dir_path / hash;
if (fs::exists(cache_hash_path)) {
// symlink hash to cache/.../hash
printout(INFO, "FileLoader", "File " + file + " with hash " + hash + " found in " + cache_hash_path.string());
try {
fs::create_symlink(cache_hash_path, hash_path);
} catch (const fs::filesystem_error&) {
printout(ERROR, "FileLoader", "unable to link from " + hash_path.string() + " to " + cache_hash_path.string());
printout(ERROR, "FileLoader", "check permissions and retry");
std::quick_exit(1);
}
break;
}
}
}
}
// if hash does not exist, we try to retrieve file from url
if (!fs::exists(hash_path)) {
cmd = fmt::format(cmd, url, hash_path.c_str()); // TODO: Use c++20 std::fmt
printout(INFO, "FileLoader", "Downloading " + file + " as hash " + hash + " with " + cmd);
// run cmd
auto ret = std::system(cmd.c_str());
if (!fs::exists(hash_path)) {
printout(ERROR, "FileLoader", "unable to run cmd " + cmd);
printout(ERROR, "FileLoader", "check command and retry");
printout(ERROR, "FileLoader", "hint: allow insecure connections with -k");
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
std::quick_exit(1);
}
}
// check if file already exists
if (fs::exists(file_path)) {
// file already exists
if (fs::is_symlink(file_path)) {
// file is symlink
if (fs::equivalent(hash_path, fs::read_symlink(file_path))) {
// link points to correct path
return;
} else {
// link points to incorrect path
if (fs::remove(file_path) == false) {
printout(ERROR, "FileLoader", "unable to remove symlink " + file_path.string());
printout(ERROR, "FileLoader", "check permissions or remove manually");
std::quick_exit(1);
}
}
} else {
// file exists but not symlink
printout(ERROR, "FileLoader", "will not remove actual file " + file_path.string());
printout(ERROR, "FileLoader", "check content, remove manually, and retry");
std::quick_exit(1);
}
}
// file_path now does not exist
// symlink file_path to hash_path
try {
// use new path from hash so file link is local
fs::create_symlink(fs::path(hash), file_path);
} catch (const fs::filesystem_error&) {
printout(ERROR, "FileLoader", "unable to link from " + file_path.string() + " to " + hash_path.string());
printout(ERROR, "FileLoader", "check permissions and retry");
std::quick_exit(1);
}
}