Mercurial > foo_out_sdl

comparison foosdk/sdk/foobar2000/SDK/metadb_handle_list.cpp @ 1:20d02a178406 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
*: check in everything else yay
author Paper <paper@tflc.us>
date Mon, 05 Jan 2026 02:15:46 -0500
parents
children
comparison
equal deleted inserted replaced
0:e9bb126753e7 1:20d02a178406
1 #include "foobar2000-sdk-pch.h"
2 #include "foosort.h"
3 #include "threadPool.h"
4 #include "foosortstring.h"
5 #include <vector>
6 #include "titleformat.h"
7 #include "library_manager.h"
8 #include "genrand.h"
9
10 namespace {
11
12 struct custom_sort_data_multi {
13 static constexpr unsigned numLocal = 4;
14 void setup(size_t count) {
15 if (count > numLocal) texts2 = std::make_unique< fb2k::sortString_t[] >( count - numLocal );
16 }
17 fb2k::sortString_t& operator[] (size_t which) {
18 return which < numLocal ? texts1[which] : texts2[which - numLocal];
19 }
20 const fb2k::sortString_t& operator[] (size_t which) const {
21 return which < numLocal ? texts1[which] : texts2[which - numLocal];
22 }
23
24 fb2k::sortString_t texts1[numLocal];
25 std::unique_ptr< fb2k::sortString_t[] > texts2;
26 size_t index;
27 };
28 struct custom_sort_data {
29 fb2k::sortString_t text;
30 size_t index;
31 };
32 template<int direction>
33 static int custom_sort_compare(const custom_sort_data& elem1, const custom_sort_data& elem2) {
34 int ret = direction * fb2k::sortStringCompare(elem1.text, elem2.text);
35 if (ret == 0) ret = pfc::sgn_t((t_ssize)elem1.index - (t_ssize)elem2.index);
36 return ret;
37 }
38
39 }
40
41 void metadb_handle_list_helper::sort_by_format(metadb_handle_list_ref p_list,const char * spec,titleformat_hook * p_hook)
42 {
43 service_ptr_t<titleformat_object> script;
44 if (titleformat_compiler::get()->compile(script,spec))
45 sort_by_format(p_list,script,p_hook);
46 }
47
48 void metadb_handle_list_helper::sort_by_format_get_order(metadb_handle_list_cref p_list,t_size* order,const char * spec,titleformat_hook * p_hook)
49 {
50 service_ptr_t<titleformat_object> script;
51 if (titleformat_compiler::get()->compile(script,spec))
52 sort_by_format_get_order(p_list,order,script,p_hook);
53 }
54
55 void metadb_handle_list_helper::sort_by_format(metadb_handle_list_ref p_list,const service_ptr_t<titleformat_object> & p_script,titleformat_hook * p_hook, int direction)
56 {
57 const t_size count = p_list.get_count();
58 pfc::array_t<t_size> order; order.set_size(count);
59 sort_by_format_get_order(p_list,order.get_ptr(),p_script,p_hook,direction);
60 p_list.reorder(order.get_ptr());
61 }
62
63 namespace {
64
65 class tfhook_sort : public titleformat_hook {
66 public:
67 tfhook_sort() {
68 m_API->seed();
69 }
70 bool process_field(titleformat_text_out *,const char *,t_size,bool &) override {
71 return false;
72 }
73 bool process_function(titleformat_text_out * p_out,const char * p_name,t_size p_name_length,titleformat_hook_function_params * p_params,bool & p_found_flag) override {
74 if (stricmp_utf8_ex(p_name, p_name_length, "rand", SIZE_MAX) == 0) {
75 t_size param_count = p_params->get_param_count();
76 t_uint32 val;
77 if (param_count == 1) {
78 t_uint32 mod = (t_uint32)p_params->get_param_uint(0);
79 if (mod > 0) {
80 val = m_API->genrand(mod);
81 } else {
82 val = 0;
83 }
84 } else {
85 val = m_API->genrand(0xFFFFFFFF);
86 }
87 p_out->write_int(titleformat_inputtypes::unknown, val);
88 p_found_flag = true;
89 return true;
90 } else {
91 return false;
92 }
93 }
94 private:
95 genrand_service::ptr m_API = genrand_service::get();
96 };
97 }
98
99 void metadb_handle_list_helper::sort_by_format_get_order(metadb_handle_list_cref p_list,t_size* order,const service_ptr_t<titleformat_object> & p_script,titleformat_hook * p_hook,int p_direction)
100 {
101 sort_by_format_get_order_v2(p_list, order, p_script, p_hook, p_direction, fb2k::noAbort );
102 }
103
104 void metadb_handle_list_helper::sort_by_relative_path(metadb_handle_list_ref p_list)
105 {
106 const t_size count = p_list.get_count();
107 pfc::array_t<t_size> order; order.set_size(count);
108 sort_by_relative_path_get_order(p_list,order.get_ptr());
109 p_list.reorder(order.get_ptr());
110 }
111
112 void metadb_handle_list_helper::sort_by_relative_path_get_order(metadb_handle_list_cref p_list,t_size* order)
113 {
114 const t_size count = p_list.get_count();
115 t_size n;
116 std::vector<custom_sort_data> data;
117 data.resize(count);
118 auto api = library_manager::get();
119
120 pfc::string8_fastalloc temp;
121 temp.prealloc(512);
122 for(n=0;n<count;n++)
123 {
124 metadb_handle_ptr item;
125 p_list.get_item_ex(item,n);
126 if (!api->get_relative_path(item,temp)) temp = "";
127 data[n].index = n;
128 data[n].text = fb2k::makeSortString(temp);
129 //data[n].subsong = item->get_subsong_index();
130 }
131
132 pfc::sort_t(data,custom_sort_compare<1>,count);
133 //qsort(data.get_ptr(),count,sizeof(custom_sort_data),(int (__cdecl *)(const void *elem1, const void *elem2 ))custom_sort_compare);
134
135 for(n=0;n<count;n++)
136 {
137 order[n]=data[n].index;
138 }
139 }
140
141 void metadb_handle_list_helper::remove_duplicates(metadb_handle_list_ref p_list)
142 {
143 t_size count = p_list.get_count();
144 if (count>0)
145 {
146 pfc::bit_array_bittable mask(count);
147 pfc::array_t<t_size> order; order.set_size(count);
148 order_helper::g_fill(order);
149
150 p_list.sort_get_permutation_t(pfc::compare_t<metadb_handle_ptr,metadb_handle_ptr>,order.get_ptr());
151
152 t_size n;
153 bool found = false;
154 for(n=0;n<count-1;n++)
155 {
156 if (p_list.get_item(order[n])==p_list.get_item(order[n+1]))
157 {
158 found = true;
159 mask.set(order[n+1],true);
160 }
161 }
162
163 if (found) p_list.remove_mask(mask);
164 }
165 }
166
167 void metadb_handle_list_helper::sort_by_pointer_remove_duplicates(metadb_handle_list_ref p_list)
168 {
169 const t_size count = p_list.get_count();
170 if (count>0)
171 {
172 sort_by_pointer(p_list);
173 bool b_found = false;
174 for(size_t n=0;n<count-1;n++)
175 {
176 if (p_list.get_item(n)==p_list.get_item(n+1))
177 {
178 b_found = true;
179 break;
180 }
181 }
182
183 if (b_found)
184 {
185 pfc::bit_array_bittable mask(count);
186 for(size_t n=0;n<count-1;n++)
187 {
188 if (p_list.get_item(n)==p_list.get_item(n+1))
189 mask.set(n+1,true);
190 }
191 p_list.remove_mask(mask);
192 }
193 }
194 }
195
196 void metadb_handle_list_helper::sort_by_path_quick(metadb_handle_list_ref p_list)
197 {
198 p_list.sort_t(metadb::path_compare_metadb_handle);
199 }
200
201
202 void metadb_handle_list_helper::sort_by_pointer(metadb_handle_list_ref p_list)
203 {
204 p_list.sort();
205 }
206
207 t_size metadb_handle_list_helper::bsearch_by_pointer(metadb_handle_list_cref p_list,const metadb_handle_ptr & val)
208 {
209 t_size blah;
210 if (p_list.bsearch_t(pfc::compare_t<metadb_handle_ptr,metadb_handle_ptr>,val,blah)) return blah;
211 else return SIZE_MAX;
212 }
213
214
215 void metadb_handle_list_helper::sorted_by_pointer_extract_difference(metadb_handle_list const & p_list_1,metadb_handle_list const & p_list_2,metadb_handle_list & p_list_1_specific,metadb_handle_list & p_list_2_specific)
216 {
217 t_size found_1, found_2;
218 const t_size count_1 = p_list_1.get_count(), count_2 = p_list_2.get_count();
219 t_size ptr_1, ptr_2;
220
221 found_1 = found_2 = 0;
222 ptr_1 = ptr_2 = 0;
223 while(ptr_1 < count_1 || ptr_2 < count_2)
224 {
225 while(ptr_1 < count_1 && (ptr_2 == count_2 || p_list_1[ptr_1] < p_list_2[ptr_2]))
226 {
227 found_1++;
228 t_size ptr_1_new = ptr_1 + 1;
229 while(ptr_1_new < count_1 && p_list_1[ptr_1_new] == p_list_1[ptr_1]) ptr_1_new++;
230 ptr_1 = ptr_1_new;
231 }
232 while(ptr_2 < count_2 && (ptr_1 == count_1 || p_list_2[ptr_2] < p_list_1[ptr_1]))
233 {
234 found_2++;
235 t_size ptr_2_new = ptr_2 + 1;
236 while(ptr_2_new < count_2 && p_list_2[ptr_2_new] == p_list_2[ptr_2]) ptr_2_new++;
237 ptr_2 = ptr_2_new;
238 }
239 while(ptr_1 < count_1 && ptr_2 < count_2 && p_list_1[ptr_1] == p_list_2[ptr_2]) {ptr_1++; ptr_2++;}
240 }
241
242
243
244 p_list_1_specific.set_count(found_1);
245 p_list_2_specific.set_count(found_2);
246 if (found_1 > 0 || found_2 > 0)
247 {
248 found_1 = found_2 = 0;
249 ptr_1 = ptr_2 = 0;
250
251 while(ptr_1 < count_1 || ptr_2 < count_2)
252 {
253 while(ptr_1 < count_1 && (ptr_2 == count_2 || p_list_1[ptr_1] < p_list_2[ptr_2]))
254 {
255 p_list_1_specific[found_1++] = p_list_1[ptr_1];
256 t_size ptr_1_new = ptr_1 + 1;
257 while(ptr_1_new < count_1 && p_list_1[ptr_1_new] == p_list_1[ptr_1]) ptr_1_new++;
258 ptr_1 = ptr_1_new;
259 }
260 while(ptr_2 < count_2 && (ptr_1 == count_1 || p_list_2[ptr_2] < p_list_1[ptr_1]))
261 {
262 p_list_2_specific[found_2++] = p_list_2[ptr_2];
263 t_size ptr_2_new = ptr_2 + 1;
264 while(ptr_2_new < count_2 && p_list_2[ptr_2_new] == p_list_2[ptr_2]) ptr_2_new++;
265 ptr_2 = ptr_2_new;
266 }
267 while(ptr_1 < count_1 && ptr_2 < count_2 && p_list_1[ptr_1] == p_list_2[ptr_2]) {ptr_1++; ptr_2++;}
268 }
269
270 }
271 }
272
273 double metadb_handle_list_helper::calc_total_duration_v2(metadb_handle_list_cref p_list, unsigned maxThreads, abort_callback & aborter) {
274 const size_t count = p_list.get_count();
275 size_t numThreads = pfc::getOptimalWorkerThreadCountEx( pfc::min_t<size_t>(maxThreads, count / 2000 ));
276 if (numThreads == 1) {
277 double ret = 0;
278 for (size_t n = 0; n < count; n++)
279 {
280 double temp = p_list.get_item(n)->get_length();
281 if (temp > 0) ret += temp;
282 }
283 return ret;
284 }
285
286 pfc::array_t<double> sums; sums.resize(numThreads); sums.fill_null();
287
288 {
289 pfc::refcounter walk = 0, walkSums = 0;
290
291 auto worker = [&] {
292 double ret = 0;
293 for (;;) {
294 size_t idx = walk++;
295 if (idx >= count || aborter.is_set()) break;
296
297 double temp = p_list.get_item(idx)->get_length();
298 if (temp > 0) ret += temp;
299 }
300 sums[walkSums++] = ret;
301 };
302
303 fb2k::cpuThreadPool::runMultiHelper(worker, numThreads);
304 }
305 aborter.check();
306 double ret = 0;
307 for (size_t walk = 0; walk < numThreads; ++walk) ret += sums[walk];
308 return ret;
309 }
310
311 pfc::string8 metadb_handle_list_helper::format_total_size(metadb_handle_list_cref p_list) {
312 pfc::string8 temp;
313 bool unknown = false;
314 t_filesize val = metadb_handle_list_helper::calc_total_size_ex(p_list,unknown);
315 if (unknown) temp << "> ";
316 temp << pfc::format_file_size_short(val);
317 return temp;
318 }
319
320 double metadb_handle_list_helper::calc_total_duration(metadb_handle_list_cref p_list)
321 {
322 double ret = 0;
323 for (auto handle : p_list) {
324 double temp = handle->get_length();
325 if (temp > 0) ret += temp;
326 }
327 return ret;
328 }
329
330 void metadb_handle_list_helper::sort_by_path(metadb_handle_list_ref p_list)
331 {
332 sort_by_format(p_list,"%path_sort%",NULL);
333 }
334
335 void metadb_handle_list_helper::sort_by_format_v2(metadb_handle_list_ref p_list, const service_ptr_t<titleformat_object> & script, titleformat_hook * hook, int direction, abort_callback & aborter) {
336 pfc::array_t<size_t> order; order.set_size( p_list.get_count() );
337 sort_by_format_get_order_v2( p_list, order.get_ptr(), script, hook, direction, aborter );
338 p_list.reorder( order.get_ptr() );
339 }
340
341 void metadb_handle_list_helper::sort_by_format_get_order_v2(metadb_handle_list_cref p_list, size_t * order, const service_ptr_t<titleformat_object> & p_script, titleformat_hook * p_hook, int p_direction, abort_callback & aborter) {
342 sorter_t s = { p_script, p_direction, p_hook };
343 size_t total = p_list.get_count();
344 for (size_t walk = 0; walk < total; ++walk) order[walk] = walk;
345 sort_by_format_get_order_v3(p_list, order, &s, 1, aborter);
346 }
347
348 void metadb_handle_list_helper::sort_by_format_get_order_v3(metadb_handle_list_cref p_list, size_t* order,sorter_t const* sorters, size_t nSorters, abort_callback& aborter) {
349 // pfc::hires_timer timer; timer.start();
350
351 typedef custom_sort_data_multi data_t;
352
353 const t_size count = p_list.get_count();
354 if (count == 0) return;
355
356 PFC_ASSERT(pfc::permutation_is_valid(order, count));
357
358 auto data = std::make_unique< data_t[] >(count);
359
360 #if FOOBAR2000_TARGET_VERSION >= 81
361 bool need_info = false;
362 for (size_t iSorter = 0; iSorter < nSorters; ++iSorter) {
363 auto& s = sorters[iSorter];
364 PFC_ASSERT(s.direction == -1 || s.direction == 1);
365 if (s.obj->requires_metadb_info_()) {
366 need_info = true; break;
367 }
368 }
369 if (need_info) {
370 // FB2K_console_formatter() << "sorting with queryMultiParallelEx_<>";
371 struct qmpc_context {
372 qmpc_context() {
373 temp.prealloc(512);
374 }
375 tfhook_sort myHook;
376 pfc::string8 temp;
377 };
378 metadb_v2::get()->queryMultiParallelEx_< qmpc_context >(p_list, [&](size_t idx, metadb_v2::rec_t const& rec, qmpc_context& ctx) {
379 aborter.check();
380 auto& out = data[idx];
381 out.setup(nSorters);
382 out.index = order[idx];
383
384 auto h = p_list[idx];
385
386 for (size_t iSorter = 0; iSorter < nSorters; ++iSorter) {
387 auto& s = sorters[iSorter];
388 if (s.hook) {
389 titleformat_hook_impl_splitter hookSplitter(&ctx.myHook, s.hook);
390 h->formatTitle_v2_(rec, &hookSplitter, ctx.temp, s.obj, nullptr);
391 } else {
392 h->formatTitle_v2_(rec, &ctx.myHook, ctx.temp, s.obj, nullptr);
393 }
394 out[iSorter] = fb2k::makeSortString(ctx.temp);
395 }
396 });
397 } else {
398 // FB2K_console_formatter() << "sorting with blank metadb info";
399 auto api = fb2k::cpuThreadPool::get();
400 pfc::counter walk = 0;
401 api->runMulti_([&] {
402 pfc::string8 temp;
403 const metadb_v2_rec_t rec = {};
404 tfhook_sort myHook;
405 for (;;) {
406 aborter.check();
407 size_t idx = walk++;
408 if (idx >= count) return;
409
410 auto& out = data[idx];
411 out.setup(nSorters);
412 out.index = order[idx];
413
414 for (size_t iSorter = 0; iSorter < nSorters; ++iSorter) {
415 auto& s = sorters[iSorter];
416 if (s.hook) {
417 titleformat_hook_impl_splitter hookSplitter(&myHook, s.hook);
418 p_list[idx]->formatTitle_v2_(rec, &hookSplitter, temp, s.obj, nullptr);
419 } else {
420 p_list[idx]->formatTitle_v2_(rec, &myHook, temp, s.obj, nullptr);
421 }
422
423 out[iSorter] = fb2k::makeSortString(temp);
424 }
425 }
426 }, api->numRunsSanity((count + 1999) / 2000));
427
428 }
429 #else
430 {
431 pfc::counter counter(0);
432
433 auto work = [&] {
434 tfhook_sort myHook;
435
436 pfc::string8_fastalloc temp; temp.prealloc(512);
437 for (;; ) {
438 const t_size index = (counter)++;
439 if (index >= count || aborter.is_set()) break;
440
441 auto& out = data[index];
442 out.setup(nSorters);
443 out.index = order[index];
444
445 for (size_t iSorter = 0; iSorter < nSorters; ++iSorter) {
446 auto& s = sorters[iSorter];
447 if (s.hook) {
448 titleformat_hook_impl_splitter hookSplitter(&myHook, s.hook);
449 p_list[index]->format_title(&hookSplitter, temp, s.obj, 0);
450 } else {
451 p_list[index]->format_title(&myHook, temp, s.obj, 0);
452 }
453
454 out[iSorter] = fb2k::makeSortString(temp);
455 }
456 }
457 };
458
459 size_t nThreads = pfc::getOptimalWorkerThreadCountEx(count / 128);
460 if (nThreads == 1) {
461 work();
462 } else {
463 fb2k::cpuThreadPool::runMultiHelper(work, nThreads);
464 }
465 }
466 #endif
467 aborter.check();
468 // console::formatter() << "metadb_handle sort: prepared in " << pfc::format_time_ex(timer.query(),6);
469
470
471 {
472 auto compare = [&](data_t const& elem1, data_t const& elem2) -> int {
473 for (size_t iSorter = 0; iSorter < nSorters; ++iSorter) {
474 int v = fb2k::sortStringCompare(elem1[iSorter], elem2[iSorter]);
475 if (v) return v * sorters[iSorter].direction;
476 }
477
478 return pfc::sgn_t((t_ssize)elem1.index - (t_ssize)elem2.index);
479 };
480
481 typedef decltype(data) container_t;
482 typedef decltype(compare) compare_t;
483 pfc::sort_callback_impl_simple_wrap_t<container_t, compare_t> cb(data, compare);
484
485 size_t concurrency = pfc::getOptimalWorkerThreadCountEx(count / 4096);
486 fb2k::sort(cb, count, concurrency, aborter);
487 }
488
489 //qsort(data.get_ptr(),count,sizeof(custom_sort_data),p_direction > 0 ? _custom_sort_compare<1> : _custom_sort_compare<-1>);
490
491
492 // console::formatter() << "metadb_handle sort: sorted in " << pfc::format_time_ex(timer.query(),6);
493
494 for (t_size n = 0; n < count; n++)
495 {
496 order[n] = data[n].index;
497 }
498
499 // FB2K_console_formatter() << "metadb_handle sort: finished in " << pfc::format_time_ex(timer.query(),6);
500
501 }
502
503 t_filesize metadb_handle_list_helper::calc_total_size(metadb_handle_list_cref p_list, bool skipUnknown) {
504 pfc::avltree_t< const char*, metadb::path_comparator > beenHere;
505 // metadb_handle_list list(p_list);
506 // list.sort_t(metadb::path_compare_metadb_handle);
507
508 t_filesize ret = 0;
509 t_size n, m = p_list.get_count();
510 for(n=0;n<m;n++) {
511 bool isNew;
512 metadb_handle_ptr h; p_list.get_item_ex(h, n);
513 beenHere.add_ex( h->get_path(), isNew);
514 if (isNew) {
515 t_filesize t = h->get_filesize();
516 if (t == filesize_invalid) {
517 if (!skipUnknown) return filesize_invalid;
518 } else {
519 ret += t;
520 }
521 }
522 }
523 return ret;
524 }
525
526 t_filesize metadb_handle_list_helper::calc_total_size_ex(metadb_handle_list_cref p_list, bool & foundUnknown) {
527 foundUnknown = false;
528 metadb_handle_list list(p_list);
529 list.sort_t(metadb::path_compare_metadb_handle);
530
531 t_filesize ret = 0;
532 t_size n, m = list.get_count();
533 for(n=0;n<m;n++) {
534 if (n==0 || metadb::path_compare(list[n-1]->get_path(),list[n]->get_path())) {
535 t_filesize t = list[n]->get_filesize();
536 if (t == filesize_invalid) {
537 foundUnknown = true;
538 } else {
539 ret += t;
540 }
541 }
542 }
543 return ret;
544 }
545
546 bool metadb_handle_list_helper::extract_folder_path(metadb_handle_list_cref list, pfc::string_base & folderOut) {
547 const t_size total = list.get_count();
548 if (total == 0) return false;
549 pfc::string_formatter temp, folder;
550 folder = list[0]->get_path();
551 folder.truncate_to_parent_path();
552 for(size_t walk = 1; walk < total; ++walk) {
553 temp = list[walk]->get_path();
554 temp.truncate_to_parent_path();
555 if (metadb::path_compare(folder, temp) != 0) return false;
556 }
557 folderOut = folder;
558 return true;
559 }
560 bool metadb_handle_list_helper::extract_single_path(metadb_handle_list_cref list, const char * &pathOut) {
561 const t_size total = list.get_count();
562 if (total == 0) return false;
563 const char * path = list[0]->get_path();
564 for(t_size walk = 1; walk < total; ++walk) {
565 if (metadb::path_compare(path, list[walk]->get_path()) != 0) return false;
566 }
567 pathOut = path;
568 return true;
569 }