channel.py 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051
  1. import asyncio
  2. import base64
  3. import gzip
  4. import json
  5. import os
  6. import pickle
  7. import re
  8. from collections import defaultdict
  9. from logging import INFO
  10. from bs4 import NavigableString
  11. import utils.constants as constants
  12. from updates.epg.tools import write_to_xml, compress_to_gz
  13. from utils.alias import Alias
  14. from utils.config import config
  15. from utils.db import get_db_connection, return_db_connection
  16. from utils.ip_checker import IPChecker
  17. from utils.speed import (
  18. get_speed,
  19. get_speed_result,
  20. get_sort_result,
  21. check_ffmpeg_installed_status
  22. )
  23. from utils.tools import (
  24. format_name,
  25. get_name_url,
  26. check_url_by_keywords,
  27. get_total_urls,
  28. add_url_info,
  29. resource_path,
  30. get_urls_from_file,
  31. get_name_urls_from_file,
  32. get_logger,
  33. get_datetime_now,
  34. get_url_host,
  35. check_ipv_type_match,
  36. get_ip_address,
  37. convert_to_m3u,
  38. custom_print,
  39. get_name_uri_from_dir, get_resolution_value
  40. )
  41. from utils.types import ChannelData, OriginType, CategoryChannelData
  42. channel_alias = Alias()
  43. ip_checker = IPChecker()
  44. frozen_channels = set()
  45. location_list = config.location
  46. isp_list = config.isp
  47. def format_channel_data(url: str, origin: OriginType) -> ChannelData:
  48. """
  49. Format the channel data
  50. """
  51. url_partition = url.partition("$")
  52. url = url_partition[0]
  53. info = url_partition[2]
  54. if info and info.startswith("!"):
  55. origin = "whitelist"
  56. info = info[1:]
  57. return {
  58. "id": hash(url),
  59. "url": url,
  60. "host": get_url_host(url),
  61. "origin": origin,
  62. "ipv_type": None,
  63. "extra_info": info
  64. }
  65. def get_channel_data_from_file(channels, file, whitelist, open_local=config.open_local,
  66. local_data=None, live_data=None, hls_data=None) -> CategoryChannelData:
  67. """
  68. Get the channel data from the file
  69. """
  70. current_category = ""
  71. for line in file:
  72. line = line.strip()
  73. if "#genre#" in line:
  74. current_category = line.partition(",")[0]
  75. else:
  76. name_url = get_name_url(
  77. line, pattern=constants.demo_txt_pattern, check_url=False
  78. )
  79. if name_url and name_url[0]:
  80. name = name_url[0]["name"]
  81. url = name_url[0]["url"]
  82. category_dict = channels[current_category]
  83. if name not in category_dict:
  84. category_dict[name] = []
  85. if name in whitelist:
  86. for whitelist_url in whitelist[name]:
  87. category_dict[name].append(format_channel_data(whitelist_url, "whitelist"))
  88. if live_data and name in live_data:
  89. for live_url in live_data[name]:
  90. category_dict[name].append(format_channel_data(live_url, "live"))
  91. if hls_data and name in hls_data:
  92. for hls_url in hls_data[name]:
  93. category_dict[name].append(format_channel_data(hls_url, "hls"))
  94. if open_local:
  95. if url:
  96. category_dict[name].append(format_channel_data(url, "local"))
  97. if local_data:
  98. format_key = format_name(name)
  99. if format_key in local_data:
  100. for local_url in local_data[format_key]:
  101. category_dict[name].append(format_channel_data(local_url, "local"))
  102. return channels
  103. def get_channel_items() -> CategoryChannelData:
  104. """
  105. Get the channel items from the source file
  106. """
  107. user_source_file = resource_path(config.source_file)
  108. channels = defaultdict(lambda: defaultdict(list))
  109. live_data = None
  110. hls_data = None
  111. if config.open_rtmp:
  112. live_data = get_name_uri_from_dir(constants.live_path)
  113. hls_data = get_name_uri_from_dir(constants.hls_path)
  114. local_data = get_name_urls_from_file(config.local_file, format_name_flag=True)
  115. whitelist = get_name_urls_from_file(constants.whitelist_path)
  116. whitelist_urls = get_urls_from_file(constants.whitelist_path)
  117. whitelist_len = len(list(whitelist.keys()))
  118. if whitelist_len:
  119. print(f"Found {whitelist_len} channel in whitelist")
  120. if os.path.exists(user_source_file):
  121. with open(user_source_file, "r", encoding="utf-8") as file:
  122. channels = get_channel_data_from_file(
  123. channels, file, whitelist, config.open_local, local_data, live_data, hls_data
  124. )
  125. if config.open_history:
  126. if os.path.exists(constants.cache_path):
  127. try:
  128. with gzip.open(constants.cache_path, "rb") as file:
  129. old_result = pickle.load(file)
  130. max_delay = config.speed_test_timeout * 1000
  131. min_resolution_value = config.min_resolution_value
  132. for cate, data in channels.items():
  133. if cate in old_result:
  134. for name, info_list in data.items():
  135. urls = [
  136. url
  137. for item in info_list
  138. if (url := item["url"])
  139. ]
  140. if name in old_result[cate]:
  141. for info in old_result[cate][name]:
  142. if info:
  143. try:
  144. delay = info.get("delay", 0)
  145. resolution = info.get("resolution")
  146. if (delay == -1 or delay > max_delay) or info.get("speed") == 0 or (
  147. resolution and get_resolution_value(
  148. resolution) < min_resolution_value):
  149. frozen_channels.add(info["url"])
  150. continue
  151. if info["origin"] == "whitelist" and not any(
  152. url in info["url"] for url in whitelist_urls):
  153. continue
  154. except:
  155. pass
  156. if info["url"] not in urls:
  157. channels[cate][name].append(info)
  158. if not channels[cate][name]:
  159. for info in old_result[cate][name]:
  160. if info and info["url"] not in urls:
  161. channels[cate][name].append(info)
  162. frozen_channels.discard(info["url"])
  163. except Exception as e:
  164. print(f"Error loading cache file: {e}")
  165. pass
  166. return channels
  167. def format_channel_name(name):
  168. """
  169. Format the channel name with sub and replace and lower
  170. """
  171. return channel_alias.get_primary(name)
  172. def channel_name_is_equal(name1, name2):
  173. """
  174. Check if the channel name is equal
  175. """
  176. name1_format = format_channel_name(name1)
  177. name2_format = format_channel_name(name2)
  178. return name1_format == name2_format
  179. def get_channel_results_by_name(name, data):
  180. """
  181. Get channel results from data by name
  182. """
  183. format_name = format_channel_name(name)
  184. results = data.get(format_name, [])
  185. return results
  186. def get_element_child_text_list(element, child_name):
  187. """
  188. Get the child text of the element
  189. """
  190. text_list = []
  191. children = element.find_all(child_name)
  192. if children:
  193. for child in children:
  194. text = child.get_text(strip=True)
  195. if text:
  196. text_list.append(text)
  197. return text_list
  198. def get_multicast_ip_list(urls):
  199. """
  200. Get the multicast ip list from urls
  201. """
  202. ip_list = []
  203. for url in urls:
  204. pattern = r"rtp://((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?::(\d+))?)"
  205. matcher = re.search(pattern, url)
  206. if matcher:
  207. ip_list.append(matcher.group(1))
  208. return ip_list
  209. def get_channel_multicast_region_ip_list(result, channel_region, channel_type):
  210. """
  211. Get the channel multicast region ip list by region and type from result
  212. """
  213. return [
  214. ip
  215. for result_region, result_obj in result.items()
  216. if result_region in channel_region
  217. for url_type, urls in result_obj.items()
  218. if url_type in channel_type
  219. for ip in get_multicast_ip_list(urls)
  220. ]
  221. def get_channel_multicast_name_region_type_result(result, names):
  222. """
  223. Get the multicast name and region and type result by names from result
  224. """
  225. name_region_type_result = {}
  226. for name in names:
  227. data = result.get(name)
  228. if data:
  229. name_region_type_result[name] = data
  230. return name_region_type_result
  231. def get_channel_multicast_region_type_list(result):
  232. """
  233. Get the channel multicast region type list from result
  234. """
  235. region_list = config.multicast_region_list
  236. region_type_list = {
  237. (region, r_type)
  238. for region_type in result.values()
  239. for region, types in region_type.items()
  240. if "all" in region_list
  241. or "ALL" in region_list
  242. or "全部" in region_list
  243. or region in region_list
  244. for r_type in types
  245. }
  246. return list(region_type_list)
  247. def get_channel_multicast_result(result, search_result):
  248. """
  249. Get the channel multicast info result by result and search result
  250. """
  251. info_result = {}
  252. multicast_name = constants.origin_map["multicast"]
  253. for name, result_obj in result.items():
  254. info_list = [
  255. {
  256. "url":
  257. add_url_info(
  258. total_url,
  259. f"{result_region}{result_type}{multicast_name}",
  260. ),
  261. "date": date,
  262. "resolution": resolution,
  263. }
  264. for result_region, result_types in result_obj.items()
  265. if result_region in search_result
  266. for result_type, result_type_urls in result_types.items()
  267. if result_type in search_result[result_region]
  268. for ip in get_multicast_ip_list(result_type_urls) or []
  269. for url, date, resolution in search_result[result_region][result_type]
  270. if (total_url := f"http://{url}/rtp/{ip}")
  271. ]
  272. info_result[name] = info_list
  273. return info_result
  274. def get_results_from_soup(soup, name):
  275. """
  276. Get the results from the soup
  277. """
  278. results = []
  279. if not soup.descendants:
  280. return results
  281. for element in soup.descendants:
  282. if isinstance(element, NavigableString):
  283. text = element.get_text(strip=True)
  284. url = get_channel_url(text)
  285. if url and not any(item[0] == url for item in results):
  286. url_element = soup.find(lambda tag: tag.get_text(strip=True) == url)
  287. if url_element:
  288. name_element = url_element.find_previous_sibling()
  289. if name_element:
  290. channel_name = name_element.get_text(strip=True)
  291. if channel_name_is_equal(name, channel_name):
  292. info_element = url_element.find_next_sibling()
  293. date, resolution = get_channel_info(
  294. info_element.get_text(strip=True)
  295. )
  296. results.append({
  297. "url": url,
  298. "date": date,
  299. "resolution": resolution,
  300. })
  301. return results
  302. def get_results_from_multicast_soup(soup, hotel=False):
  303. """
  304. Get the results from the multicast soup
  305. """
  306. results = []
  307. if not soup.descendants:
  308. return results
  309. for element in soup.descendants:
  310. if isinstance(element, NavigableString):
  311. text = element.strip()
  312. if "失效" in text:
  313. continue
  314. url = get_channel_url(text)
  315. if url and not any(item["url"] == url for item in results):
  316. url_element = soup.find(lambda tag: tag.get_text(strip=True) == url)
  317. if not url_element:
  318. continue
  319. parent_element = url_element.find_parent()
  320. info_element = parent_element.find_all(recursive=False)[-1]
  321. if not info_element:
  322. continue
  323. info_text = info_element.get_text(strip=True)
  324. if "上线" in info_text and " " in info_text:
  325. date, region, channel_type = get_multicast_channel_info(info_text)
  326. if hotel and "酒店" not in region:
  327. continue
  328. results.append(
  329. {
  330. "url": url,
  331. "date": date,
  332. "region": region,
  333. "type": channel_type,
  334. }
  335. )
  336. return results
  337. def get_results_from_soup_requests(soup, name):
  338. """
  339. Get the results from the soup by requests
  340. """
  341. results = []
  342. elements = soup.find_all("div", class_="resultplus") if soup else []
  343. for element in elements:
  344. name_element = element.find("div", class_="channel")
  345. if name_element:
  346. channel_name = name_element.get_text(strip=True)
  347. if channel_name_is_equal(name, channel_name):
  348. text_list = get_element_child_text_list(element, "div")
  349. url = date = resolution = None
  350. for text in text_list:
  351. text_url = get_channel_url(text)
  352. if text_url:
  353. url = text_url
  354. if " " in text:
  355. text_info = get_channel_info(text)
  356. date, resolution = text_info
  357. if url:
  358. results.append({
  359. "url": url,
  360. "date": date,
  361. "resolution": resolution,
  362. })
  363. return results
  364. def get_results_from_multicast_soup_requests(soup, hotel=False):
  365. """
  366. Get the results from the multicast soup by requests
  367. """
  368. results = []
  369. if not soup:
  370. return results
  371. elements = soup.find_all("div", class_="result")
  372. for element in elements:
  373. name_element = element.find("div", class_="channel")
  374. if not name_element:
  375. continue
  376. text_list = get_element_child_text_list(element, "div")
  377. url, date, region, channel_type = None, None, None, None
  378. valid = True
  379. for text in text_list:
  380. if "失效" in text:
  381. valid = False
  382. break
  383. text_url = get_channel_url(text)
  384. if text_url:
  385. url = text_url
  386. if url and "上线" in text and " " in text:
  387. date, region, channel_type = get_multicast_channel_info(text)
  388. if url and valid:
  389. if hotel and "酒店" not in region:
  390. continue
  391. results.append({"url": url, "date": date, "region": region, "type": channel_type})
  392. return results
  393. def get_channel_url(text):
  394. """
  395. Get the url from text
  396. """
  397. url = None
  398. url_search = constants.url_pattern.search(text)
  399. if url_search:
  400. url = url_search.group()
  401. return url
  402. def get_channel_info(text):
  403. """
  404. Get the channel info from text
  405. """
  406. date, resolution = None, None
  407. if text:
  408. date, resolution = (
  409. (text.partition(" ")[0] if text.partition(" ")[0] else None),
  410. (
  411. text.partition(" ")[2].partition("•")[2]
  412. if text.partition(" ")[2].partition("•")[2]
  413. else None
  414. ),
  415. )
  416. return date, resolution
  417. def get_multicast_channel_info(text):
  418. """
  419. Get the multicast channel info from text
  420. """
  421. date, region, channel_type = None, None, None
  422. if text:
  423. text_split = text.split(" ")
  424. filtered_data = list(filter(lambda x: x.strip() != "", text_split))
  425. if filtered_data and len(filtered_data) == 4:
  426. date = filtered_data[0]
  427. region = filtered_data[2]
  428. channel_type = filtered_data[3]
  429. return date, region, channel_type
  430. def init_info_data(data: dict, category: str, name: str) -> None:
  431. """
  432. Initialize channel info data structure if not exists
  433. """
  434. data.setdefault(category, {}).setdefault(name, [])
  435. def append_data_to_info_data(
  436. info_data: dict,
  437. category: str,
  438. name: str,
  439. data: list,
  440. origin: str = None,
  441. check: bool = True,
  442. whitelist: list = None,
  443. blacklist: list = None,
  444. ipv_type_data: dict = None
  445. ) -> None:
  446. """
  447. Append channel data to total info data with deduplication and validation
  448. Args:
  449. info_data: The main data structure to update
  450. category: Category key for the data
  451. name: Name key within the category
  452. data: List of channel items to process
  453. origin: Default origin for items
  454. check: Whether to perform validation checks
  455. whitelist: List of whitelist keywords
  456. blacklist: List of blacklist keywords
  457. ipv_type_data: Dictionary to cache IP type information
  458. """
  459. init_info_data(info_data, category, name)
  460. channel_list = info_data[category][name]
  461. existing_urls = {info["url"] for info in channel_list if "url" in info}
  462. for item in data:
  463. try:
  464. channel_id = item.get("id") or hash(item["url"])
  465. url = item["url"]
  466. host = item.get("host") or get_url_host(url)
  467. date = item.get("date")
  468. delay = item.get("delay")
  469. speed = item.get("speed")
  470. resolution = item.get("resolution")
  471. url_origin = item.get("origin", origin)
  472. ipv_type = item.get("ipv_type")
  473. location = item.get("location")
  474. isp = item.get("isp")
  475. headers = item.get("headers")
  476. catchup = item.get("catchup")
  477. extra_info = item.get("extra_info", "")
  478. if not url_origin or not url:
  479. continue
  480. if url in frozen_channels or (url in existing_urls and (url_origin != "whitelist" and not headers)):
  481. continue
  482. if not ipv_type:
  483. if ipv_type_data and host in ipv_type_data:
  484. ipv_type = ipv_type_data[host]
  485. else:
  486. ipv_type = ip_checker.get_ipv_type(url)
  487. if ipv_type_data is not None:
  488. ipv_type_data[host] = ipv_type
  489. if not location or not isp:
  490. ip = ip_checker.get_ip(url)
  491. if ip:
  492. location, isp = ip_checker.find_map(ip)
  493. if location and location_list and not any(item in location for item in location_list):
  494. continue
  495. if isp and isp_list and not any(item in isp for item in isp_list):
  496. continue
  497. for idx, info in enumerate(info_data[category][name]):
  498. if not info.get("url"):
  499. continue
  500. info_host = get_url_host(info["url"])
  501. if info_host == host:
  502. info_url = info["url"]
  503. # Replace if new URL is shorter or has headers
  504. if len(info_url) > len(url) or headers:
  505. if url in existing_urls:
  506. existing_urls.remove(url)
  507. existing_urls.add(info_url)
  508. info_data[category][name][idx] = {
  509. "id": channel_id,
  510. "url": info_url,
  511. "host": host,
  512. "date": date,
  513. "delay": delay,
  514. "speed": speed,
  515. "resolution": resolution,
  516. "origin": origin,
  517. "ipv_type": ipv_type,
  518. "location": location,
  519. "isp": isp,
  520. "headers": headers,
  521. "catchup": catchup,
  522. "extra_info": extra_info
  523. }
  524. break
  525. continue
  526. if whitelist and check_url_by_keywords(url, whitelist):
  527. url_origin = "whitelist"
  528. if (not check or
  529. url_origin in ["whitelist", "live", "hls"] or
  530. (check_ipv_type_match(ipv_type) and
  531. not check_url_by_keywords(url, blacklist))):
  532. channel_list.append({
  533. "id": channel_id,
  534. "url": url,
  535. "host": host,
  536. "date": date,
  537. "delay": delay,
  538. "speed": speed,
  539. "resolution": resolution,
  540. "origin": url_origin,
  541. "ipv_type": ipv_type,
  542. "location": location,
  543. "isp": isp,
  544. "headers": headers,
  545. "catchup": catchup,
  546. "extra_info": extra_info
  547. })
  548. existing_urls.add(url)
  549. except Exception as e:
  550. print(f"Error processing channel data: {e}")
  551. continue
  552. def get_origin_method_name(method):
  553. """
  554. Get the origin method name
  555. """
  556. return "hotel" if method.startswith("hotel_") else method
  557. def append_old_data_to_info_data(info_data, cate, name, data, whitelist=None, blacklist=None, ipv_type_data=None):
  558. """
  559. Append history and local channel data to total info data
  560. """
  561. append_data_to_info_data(
  562. info_data,
  563. cate,
  564. name,
  565. data,
  566. whitelist=whitelist,
  567. blacklist=blacklist,
  568. ipv_type_data=ipv_type_data
  569. )
  570. live_len = sum(1 for item in data if item["origin"] == "live")
  571. hls_len = sum(1 for item in data if item["origin"] == "hls")
  572. local_len = sum(1 for item in data if item["origin"] == "local")
  573. whitelist_len = sum(1 for item in data if item["origin"] == "whitelist")
  574. history_len = len(data) - (live_len + hls_len + local_len + whitelist_len)
  575. print(f"History: {history_len}, Live: {live_len}, HLS: {hls_len}, Local: {local_len}, Whitelist: {whitelist_len}",
  576. end=", ")
  577. def print_channel_number(data: CategoryChannelData, cate: str, name: str):
  578. """
  579. Print channel number
  580. """
  581. channel_list = data.get(cate, {}).get(name, [])
  582. print("IPv4:", len([channel for channel in channel_list if channel["ipv_type"] == "ipv4"]), end=", ")
  583. print("IPv6:", len([channel for channel in channel_list if channel["ipv_type"] == "ipv6"]), end=", ")
  584. print(
  585. "Total:",
  586. len(channel_list),
  587. )
  588. def append_total_data(
  589. items,
  590. data,
  591. hotel_fofa_result=None,
  592. multicast_result=None,
  593. hotel_foodie_result=None,
  594. subscribe_result=None,
  595. online_search_result=None,
  596. ):
  597. """
  598. Append all method data to total info data
  599. """
  600. total_result = [
  601. ("hotel_fofa", hotel_fofa_result),
  602. ("multicast", multicast_result),
  603. ("hotel_foodie", hotel_foodie_result),
  604. ("subscribe", subscribe_result),
  605. ("online_search", online_search_result),
  606. ]
  607. whitelist = get_urls_from_file(constants.whitelist_path)
  608. blacklist = get_urls_from_file(constants.blacklist_path, pattern_search=False)
  609. url_hosts_ipv_type = {}
  610. open_history = config.open_history
  611. open_local = config.open_local
  612. open_rtmp = config.open_rtmp
  613. for obj in data.values():
  614. for value_list in obj.values():
  615. for value in value_list:
  616. if value_ipv_type := value.get("ipv_type", None):
  617. url_hosts_ipv_type[get_url_host(value["url"])] = value_ipv_type
  618. for cate, channel_obj in items:
  619. for name, old_info_list in channel_obj.items():
  620. print(f"{name}:", end=" ")
  621. if (open_history or open_local or open_rtmp) and old_info_list:
  622. append_old_data_to_info_data(data, cate, name, old_info_list, whitelist=whitelist, blacklist=blacklist,
  623. ipv_type_data=url_hosts_ipv_type)
  624. for method, result in total_result:
  625. if config.open_method[method]:
  626. origin_method = get_origin_method_name(method)
  627. if not origin_method:
  628. continue
  629. name_results = get_channel_results_by_name(name, result)
  630. append_data_to_info_data(
  631. data, cate, name, name_results, origin=origin_method, whitelist=whitelist, blacklist=blacklist,
  632. ipv_type_data=url_hosts_ipv_type
  633. )
  634. print(f"{method.capitalize()}:", len(name_results), end=", ")
  635. print_channel_number(data, cate, name)
  636. async def test_speed(data, ipv6=False, callback=None):
  637. """
  638. Test speed of channel data
  639. """
  640. ipv6_proxy_url = None if (not config.open_ipv6 or ipv6) else constants.ipv6_proxy
  641. open_headers = config.open_headers
  642. get_resolution = config.open_filter_resolution and check_ffmpeg_installed_status()
  643. semaphore = asyncio.Semaphore(config.speed_test_limit)
  644. async def limited_get_speed(channel_info):
  645. """
  646. Wrapper for get_speed with rate limiting
  647. """
  648. async with semaphore:
  649. headers = (open_headers and channel_info.get("headers")) or None
  650. return await get_speed(
  651. channel_info,
  652. headers=headers,
  653. ipv6_proxy=ipv6_proxy_url,
  654. filter_resolution=get_resolution,
  655. callback=callback,
  656. )
  657. tasks = []
  658. channel_map = {}
  659. for cate, channel_obj in data.items():
  660. for name, info_list in channel_obj.items():
  661. for info in info_list:
  662. task = asyncio.create_task(limited_get_speed(info))
  663. tasks.append(task)
  664. channel_map[task] = (cate, name, info)
  665. results = await asyncio.gather(*tasks)
  666. grouped_results = {}
  667. for task, result in zip(tasks, results):
  668. cate, name, info = channel_map[task]
  669. if cate not in grouped_results:
  670. grouped_results[cate] = {}
  671. if name not in grouped_results[cate]:
  672. grouped_results[cate][name] = []
  673. grouped_results[cate][name].append({**info, **result})
  674. return grouped_results
  675. def sort_channel_result(channel_data, result=None, filter_host=False, ipv6_support=True):
  676. """
  677. Sort channel result
  678. """
  679. channel_result = defaultdict(lambda: defaultdict(list))
  680. logger = get_logger(constants.result_log_path, level=INFO, init=True)
  681. for cate, obj in channel_data.items():
  682. for name, values in obj.items():
  683. if not values:
  684. continue
  685. whitelist_result = []
  686. test_result = result.get(cate, {}).get(name, []) if result else []
  687. for value in values:
  688. if value["origin"] in ["whitelist", "live", "hls"] or (
  689. not ipv6_support and result and value["ipv_type"] == "ipv6"
  690. ):
  691. whitelist_result.append(value)
  692. elif filter_host or not result:
  693. test_result.append({**value, **get_speed_result(value["host"])} if filter_host else value)
  694. total_result = whitelist_result + get_sort_result(test_result, ipv6_support=ipv6_support)
  695. append_data_to_info_data(
  696. channel_result,
  697. cate,
  698. name,
  699. total_result,
  700. check=False,
  701. )
  702. for item in total_result:
  703. logger.info(
  704. f"Name: {name}, URL: {item.get('url')}, IPv_Type: {item.get("ipv_type")}, Location: {item.get('location')}, ISP: {item.get('isp')}, Date: {item["date"]}, Delay: {item.get('delay') or -1} ms, Speed: {item.get('speed') or 0:.2f} M/s, Resolution: {item.get('resolution')}"
  705. )
  706. logger.handlers.clear()
  707. return channel_result
  708. def process_write_content(
  709. path: str,
  710. data: CategoryChannelData,
  711. live: bool = False,
  712. hls: bool = False,
  713. live_url: str = None,
  714. hls_url: str = None,
  715. open_empty_category: bool = False,
  716. ipv_type_prefer: list[str] = None,
  717. origin_type_prefer: list[str] = None,
  718. first_channel_name: str = None,
  719. enable_print: bool = False
  720. ):
  721. """
  722. Get channel write content
  723. :param path: write into path
  724. :param live: all live channel url
  725. :param hls: all hls channel url
  726. :param live_url: live url
  727. :param hls_url: hls url
  728. :param open_empty_category: show empty category
  729. :param ipv_type_prefer: ipv type prefer
  730. :param origin_type_prefer: origin type prefer
  731. :param first_channel_name: the first channel name
  732. """
  733. content = ""
  734. no_result_name = []
  735. first_cate = True
  736. result_data = defaultdict(list)
  737. custom_print.disable = not enable_print
  738. rtmp_url = live_url if live else hls_url if hls else None
  739. rtmp_type = ["live", "hls"] if live and hls else ["live"] if live else ["hls"] if hls else []
  740. open_url_info = config.open_url_info
  741. for cate, channel_obj in data.items():
  742. custom_print(f"\n{cate}:", end=" ")
  743. content += f"{'\n\n' if not first_cate else ''}{cate},#genre#"
  744. first_cate = False
  745. channel_obj_keys = channel_obj.keys()
  746. names_len = len(list(channel_obj_keys))
  747. for i, name in enumerate(channel_obj_keys):
  748. info_list = data.get(cate, {}).get(name, [])
  749. channel_urls = get_total_urls(info_list, ipv_type_prefer, origin_type_prefer, rtmp_type)
  750. result_data[name].extend(channel_urls)
  751. end_char = ", " if i < names_len - 1 else ""
  752. custom_print(f"{name}:", len(channel_urls), end=end_char)
  753. if not channel_urls:
  754. if open_empty_category:
  755. no_result_name.append(name)
  756. continue
  757. for item in channel_urls:
  758. item_origin = item.get("origin", None)
  759. item_rtmp_url = None
  760. if item_origin == "live":
  761. item_rtmp_url = live_url
  762. elif item_origin == "hls":
  763. item_rtmp_url = hls_url
  764. item_url = item["url"]
  765. if open_url_info and item["extra_info"]:
  766. item_url = add_url_info(item_url, item["extra_info"])
  767. total_item_url = f"{rtmp_url or item_rtmp_url}{item['id']}" if rtmp_url or item_rtmp_url else item_url
  768. content += f"\n{name},{total_item_url}"
  769. custom_print()
  770. if open_empty_category and no_result_name:
  771. custom_print("\n🈳 No result channel name:")
  772. content += "\n\n🈳无结果频道,#genre#"
  773. for i, name in enumerate(no_result_name):
  774. end_char = ", " if i < len(no_result_name) - 1 else ""
  775. custom_print(name, end=end_char)
  776. content += f"\n{name},url"
  777. custom_print()
  778. if config.open_update_time:
  779. update_time_item = next(
  780. (urls[0] for channel_obj in data.values()
  781. for info_list in channel_obj.values()
  782. if (urls := get_total_urls(info_list, ipv_type_prefer, origin_type_prefer, rtmp_type))),
  783. {"id": "id", "url": "url"}
  784. )
  785. now = get_datetime_now()
  786. update_time_item_url = update_time_item["url"]
  787. if open_url_info and update_time_item["extra_info"]:
  788. update_time_item_url = add_url_info(update_time_item_url, update_time_item["extra_info"])
  789. value = f"{rtmp_url}{update_time_item["id"]}" if rtmp_url else update_time_item_url
  790. if config.update_time_position == "top":
  791. content = f"🕘️更新时间,#genre#\n{now},{value}\n\n{content}"
  792. else:
  793. content += f"\n\n🕘️更新时间,#genre#\n{now},{value}"
  794. if rtmp_url:
  795. conn = get_db_connection(constants.rtmp_data_path)
  796. try:
  797. cursor = conn.cursor()
  798. cursor.execute(
  799. "CREATE TABLE IF NOT EXISTS result_data (id TEXT PRIMARY KEY, url TEXT, headers TEXT)"
  800. )
  801. for data_list in result_data.values():
  802. for item in data_list:
  803. cursor.execute(
  804. "INSERT OR REPLACE INTO result_data (id, url, headers) VALUES (?, ?, ?)",
  805. (item["id"], item["url"], json.dumps(item.get("headers", None)))
  806. )
  807. conn.commit()
  808. finally:
  809. return_db_connection(constants.rtmp_data_path, conn)
  810. with open(path, "w", encoding="utf-8") as f:
  811. f.write(content)
  812. convert_to_m3u(path, first_channel_name, data=result_data)
  813. def write_channel_to_file(data, epg=None, ipv6=False, first_channel_name=None):
  814. """
  815. Write channel to file
  816. """
  817. try:
  818. print("Write channel to file...")
  819. output_dir = constants.output_dir
  820. dir_list = [
  821. output_dir,
  822. f"{output_dir}/epg",
  823. f"{output_dir}/ipv4",
  824. f"{output_dir}/ipv6",
  825. f"{output_dir}/data",
  826. f"{output_dir}/log",
  827. ]
  828. for dir_name in dir_list:
  829. os.makedirs(dir_name, exist_ok=True)
  830. if epg:
  831. write_to_xml(epg, constants.epg_result_path)
  832. compress_to_gz(constants.epg_result_path, constants.epg_gz_result_path)
  833. open_empty_category = config.open_empty_category
  834. ipv_type_prefer = list(config.ipv_type_prefer)
  835. if any(pref in ipv_type_prefer for pref in ["自动", "auto"]):
  836. ipv_type_prefer = ["ipv6", "ipv4"] if ipv6 else ["ipv4", "ipv6"]
  837. origin_type_prefer = config.origin_type_prefer
  838. address = get_ip_address()
  839. live_url = f"{address}/live/"
  840. hls_url = f"{address}/hls/"
  841. file_list = [
  842. {"path": config.final_file, "enable_log": True},
  843. {"path": constants.ipv4_result_path, "ipv_type_prefer": ["ipv4"]},
  844. {"path": constants.ipv6_result_path, "ipv_type_prefer": ["ipv6"]}
  845. ]
  846. if config.open_rtmp and not os.getenv("GITHUB_ACTIONS"):
  847. file_list += [
  848. {"path": constants.live_result_path, "live": True},
  849. {
  850. "path": constants.live_ipv4_result_path,
  851. "live": True,
  852. "ipv_type_prefer": ["ipv4"]
  853. },
  854. {
  855. "path": constants.live_ipv6_result_path,
  856. "live": True,
  857. "ipv_type_prefer": ["ipv6"]
  858. },
  859. {"path": constants.hls_result_path, "hls": True},
  860. {
  861. "path": constants.hls_ipv4_result_path,
  862. "hls": True,
  863. "ipv_type_prefer": ["ipv4"]
  864. },
  865. {
  866. "path": constants.hls_ipv6_result_path,
  867. "hls": True,
  868. "ipv_type_prefer": ["ipv6"]
  869. },
  870. ]
  871. for file in file_list:
  872. process_write_content(
  873. path=file["path"],
  874. data=data,
  875. live=file.get("live", False),
  876. hls=file.get("hls", False),
  877. live_url=live_url,
  878. hls_url=hls_url,
  879. open_empty_category=open_empty_category,
  880. ipv_type_prefer=file.get("ipv_type_prefer", ipv_type_prefer),
  881. origin_type_prefer=origin_type_prefer,
  882. first_channel_name=first_channel_name,
  883. enable_print=file.get("enable_log", False),
  884. )
  885. print("✅ Write channel to file success")
  886. except Exception as e:
  887. print(f"❌ Write channel to file failed: {e}")
  888. def get_multicast_fofa_search_org(region, org_type):
  889. """
  890. Get the fofa search organization for multicast
  891. """
  892. org = None
  893. if region == "北京" and org_type == "联通":
  894. org = "China Unicom Beijing Province Network"
  895. elif org_type == "联通":
  896. org = "CHINA UNICOM China169 Backbone"
  897. elif org_type == "电信":
  898. org = "Chinanet"
  899. elif org_type == "移动":
  900. org = "China Mobile communications corporation"
  901. return org
  902. def get_multicast_fofa_search_urls():
  903. """
  904. Get the fofa search urls for multicast
  905. """
  906. rtp_file_names = []
  907. for filename in os.listdir(resource_path("config/rtp")):
  908. if filename.endswith(".txt") and "_" in filename:
  909. filename = filename.replace(".txt", "")
  910. rtp_file_names.append(filename)
  911. region_list = config.multicast_region_list
  912. region_type_list = [
  913. (parts[0], parts[1])
  914. for name in rtp_file_names
  915. if (parts := name.partition("_"))[0] in region_list
  916. or "all" in region_list
  917. or "ALL" in region_list
  918. or "全部" in region_list
  919. ]
  920. search_urls = []
  921. for region, r_type in region_type_list:
  922. search_url = "https://fofa.info/result?qbase64="
  923. search_txt = f'"udpxy" && country="CN" && region="{region}" && org="{get_multicast_fofa_search_org(region, r_type)}"'
  924. bytes_string = search_txt.encode("utf-8")
  925. search_txt = base64.b64encode(bytes_string).decode("utf-8")
  926. search_url += search_txt
  927. search_urls.append((search_url, region, r_type))
  928. return search_urls
  929. def get_channel_data_cache_with_compare(data, new_data):
  930. """
  931. Get channel data with cache compare new data
  932. """
  933. for cate, obj in new_data.items():
  934. for name, url_info in obj.items():
  935. if url_info and cate in data and name in data[cate]:
  936. new_urls = {
  937. info["url"]: info["resolution"]
  938. for info in url_info
  939. }
  940. updated_data = []
  941. for info in data[cate][name]:
  942. url = info["url"]
  943. if url in new_urls:
  944. resolution = new_urls[url]
  945. updated_data.append({
  946. "id": info["id"],
  947. "url": url,
  948. "date": info["date"],
  949. "resolution": resolution,
  950. "origin": info["origin"],
  951. "ipv_type": info["ipv_type"]
  952. })
  953. data[cate][name] = updated_data