A Case Study on the Efficacy of Machine Learning Algorithms in Predicting Sepsis Outcomes in Hospitalized Patients
Keywords:
Sepsis prediction, Machine learning, Clinical algorithms, Patient outcomes, Critical care, Data analytics, Healthcare innovation, Predictive modeling, Medical informaticsAbstract
Sepsis remains a leading cause of morbidity and mortality in critically ill patients, demanding innovative strategies for timely intervention. This study aims to evaluate the efficacy of machine learning algorithms in predicting sepsis outcomes, leveraging clinical data from a cohort of hospitalized patients. A total of 1,500 patient records were analyzed using supervised learning models, including Random Forest and Gradient Boosting. Results indicate that the models demonstrated significant predictive accuracy, with an area under the curve (AUC) of 0.85, surpassing traditional scoring systems such as SOFA and qSOFA. These findings suggest that integrating machine learning into clinical practice could enhance early detection and treatment of sepsis, potentially reducing mortality rates. This study highlights the critical role of advanced analytics in modern medicine, advocating for broader implementation across healthcare systems.
References
import logging
import time
from urllib.parse import urljoin, urlparse
from datetime import datetime
import xml.etree.ElementTree as ET
import xml.dom.minidom
import re
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
from bs4 import BeautifulSoup
# --- КОНФИГУРАЦИЯ ---
SITE_CONFIG = {
'humstudios.com.ua': {
'base_url': 'https://humstudios.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'div.content',
'issue_date_container': 'div.event_date',
'journal_title_fallback': 'h1',
'issn_container': 'div.h_bot_part div.col-xs-12',
'article_item': 'div.section:has(h3.section_title)',
'article_link': 'h3.section_title a',
'article_doi_text': 'li.website:-soup-contains("DOI")',
'article_details_waitFor': 'h2.section_title',
'article_title': 'h2.section_title',
'authors_container': 'h2.section_title',
'author_link': 'a[href*="/en/author/"]',
'abstract_container': 'iframe[class="iframe-responsive"]',
'dates_container': 'div.event_date',
'pages_container': 'div.text_post_section.tags_section div.btn-tags',
'article_doi_link': 'p.button.banner_button',
'article_pdf_iframe_src': 'iframe[src*="/web/uploads/pdf/"]',
},
'patterns': {
'volume_issue_url_part': 't-',
'doi_prefix_separator': '/',
'date_format': '%d.%m.%Y',
'revised_date_keyword': 'Revised',
'accepted_date_keyword': 'Accepted',
'received_date_keyword': 'Received',
'pages_keyword': 'Pages',
'default_doi_prefix': '10.0000',
},
'lang': 'en'
},
'd-governance.com.ua': {
'base_url': 'https://d-governance.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'div.attorney-single',
'issue_title': 'div.info-header h3',
'issue_date': 'ul.contact-info li',
'journal_title': 'header.main-header h1',
'issn_footer': 'footer.main-footer .footer-block .text',
'article_item': 'article.featured-news-column',
'article_link': 'h3 a',
'article_details_waitFor': 'h3.article-title',
'article_title': 'h3.article-title',
'authors_container': 'div.article-authors a.article-author',
'abstract_block': 'div.article-info-block',
'dates_block': 'div.article-info-block.additional-block',
},
'patterns': {
'volume_issue_regex': r'Volumes*(d+),s*No.s*(d+)',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.56318',
'journal_abbrev': 'dg',
},
'lang': 'en'
},
'lawjournal.com.ua': {
'base_url': 'https://lawjournal.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'section.details_section',
'issue_title': 'ul.breadcrumb_nav h2',
'issue_date': 'ul.course_info_list li',
'journal_title': 'h1.banner_big_title',
'issn_banner': 'h2.banner_small_title',
'article_item': 'div.course_card.list_layout',
'article_link': 'h3.item_title a',
'article_details_waitFor': 'section.page_banner',
},
'patterns': {
'volume_issue_regex': r'Volumes*(d+),s*No.s*(d+)',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.63341',
'journal_abbrev': 'naia-chasopis',
'issn_print': '2519-4216',
'eissn': '2519-4313',
},
'lang': 'en'
},
'journal-knu.com.ua': {
'base_url': 'https://journal-knu.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'section.journal-issue-redesign',
'issue_title': 'h3.journal-title',
'issue_date': 'span.journal-date',
'issue_doi': 'span.journal-doi',
'journal_title': 'h1.banner_big_title',
'issn_block': 'div.journal-info-block p',
'article_item': 'article.article-card',
'article_link': 'h3.article-title a',
'article_details_waitFor': 'main.article-main-redesign',
},
'patterns': {
'volume_issue_regex': r'Volumes*(d+),s*No.s*(d+)',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.31721/2306-5451',
'journal_abbrev': 'jkrnu',
'issn_print': '2306-5451',
'eissn': '2519-2736',
},
'lang': 'en'
},
'devma.com.ua': {
'base_url': 'https://devma.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'div.attorney-single',
'issue_title': 'div.info-header h3',
'issue_date': 'ul.contact-info li',
'journal_title': 'div.title',
'issn_container': 'div.issn',
'article_item': 'article.featured-news-column',
'article_link': 'h3 a',
'article_details_waitFor': 'div.details-header.article',
},
'patterns': {
'volume_issue_regex': r'Volumes*(d+),s*No.s*(d+)',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.57111',
'journal_abbrev': 'devt',
'issn_print': '2413-9610',
'eissn': '2663-2365',
},
'lang': 'en'
},
'ue-bulletin.com.ua': {
'base_url': 'https://ue-bulletin.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'div.issue-page',
'issue_title': 'h1.issue-page__title',
'issue_date': 'div.issue-page__date',
'journal_title': 'h1.site-header__brand-title',
'issn_container': 'div.site-header__meta',
'article_item': 'article.issue-articles__item',
'article_link': 'h3.issue-articles__item-title a',
'article_doi': 'a.issue-articles__doi-link',
'article_details_waitFor': 'header.article-page__header',
},
'patterns': {
'volume_issue_regex': r'Vol.s*(d+),s*No.s*(d+)',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.69587',
'journal_abbrev': 'ueb',
'issn_print': '2306-546X',
'eissn': '2414-3774',
'doi_pattern': r'10.d+/w+/d+.d+.d+',
},
'lang': 'en'
},
'profedu.com.ua': {
'base_url': 'https://profedu.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'div.issue-page',
'issue_title': 'h1.issue-page__title',
'issue_date': 'div.issue-page__date',
'journal_title': 'h1.site-header__brand-title',
'issn_container': 'div.site-header__meta',
'article_item': 'article.issue-articles__item',
'article_link': 'h3.issue-articles__item-title a',
'article_doi': 'a.issue-articles__doi-link',
'article_details_waitFor': 'header.article-page__header',
},
'patterns': {
'volume_issue_regex': r'Vol.s*(d+),s*No.s*(d+)',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.69587',
'journal_abbrev': 'pemtt',
'issn_print': '2415-3729',
'eissn': '2415-3737',
'doi_pattern': r'10.d+/w+/d+.d+.d+',
},
'lang': 'en'
},
'arrcjournal.org': {
'base_url': 'https://arrcjournal.org/en/journals/',
'selectors': {
'issue_container_waitFor': 'div.attorney-single',
'issue_title': 'h3.custom-member-title',
'issue_date': 'li.custom-contact-date',
'journal_title': 'div.widget-about p strong',
'issn_container': 'div.site-header__meta',
'article_item': 'article.widget-category.featured-news-column',
'article_link': 'h3.custom-article-title a',
'article_doi': 'p.custom-article-doi a',
'article_details_waitFor': 'div.details-header.default-text-box',
},
'patterns': {
'volume_issue_regex': r'Iss.s*(d+),s*(d{4})',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.33744',
'journal_abbrev': 'arrc',
'issn_print': '0365-8171',
'eissn': '2707-4099',
'doi_pattern': r'10.d+/w+-d+-d+.d+',
},
'lang': 'en'
},
'artnotes.com.ua': {
'base_url': 'https://artnotes.com.ua/en/journals/',
'selectors': {
'issue_container_waitFor': 'section.issue-page',
'issue_title': 'h1.issue-page__title',
'issue_date': 'span.issue-page__meta-item',
'journal_title': 'span.journal-brand__title',
'issn_container': 'div.journal-header__issn',
'article_item': 'article.issue-article',
'article_link': 'h3.issue-article__title a',
'article_doi': 'p.issue-article__doi a',
'article_details_waitFor': 'header.article-hero',
},
'patterns': {
'volume_issue_regex': r'Volumes*(d+),s*No.s*(d+),s*(d{4})',
'date_format': '%d.%m.%Y',
'default_doi_prefix': '10.63009',
'journal_abbrev': 'noac',
'issn_print': '2226-2180',
'eissn': '2226-2180',
'doi_pattern': r'10.d+/w+/d+.d+.d+',
},
'lang': 'en'
},
}
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
def sanitize_xml_string(text_to_clean):
"""Удаляет недопустимые в XML 1.0 символы из строки."""
if not isinstance(text_to_clean, str):
return text_to_clean
illegal_xml_chars_re = re.compile(r'[x00-x08x0bx0cx0e-x1fx7f-x84x86-x9f]')
return illegal_xml_chars_re.sub('', text_to_clean)
def get_config_for_url(url):
parsed_url = urlparse(url)
domain = parsed_url.netloc.replace('www.', '')
if domain in SITE_CONFIG:
logging.info(f"Используется конфигурация для: {domain}")
return SITE_CONFIG[domain]
else:
parts = domain.split('.')
for i in range(len(parts) - 1):
parent_domain = '.'.join(parts[i:])
if parent_domain in SITE_CONFIG:
logging.info(f"Используется конфигурация для родительского домена: {parent_domain} (для {domain})")
return SITE_CONFIG[parent_domain]
logging.error(f"Конфигурация для домена {domain} не найдена!")
raise ValueError(f"Конфигурация для домена {domain} не найдена!")
def parse_date(date_str, fmt, default_year):
if date_str is None:
return {'month': '01', 'day': '01', 'year': str(default_year)}
try:
date_obj = datetime.strptime(date_str.strip(), fmt)
return {
'month': date_obj.strftime('%m'),
'day': date_obj.strftime('%d'),
'year': date_obj.strftime('%Y')
}
except (ValueError, TypeError):
return {'month': '01', 'day': '01', 'year': str(default_year)}
def safe_get_text(element, default=""):
return element.get_text(strip=True) if element else default
def find_element(soup, selectors, first_only=True):
if not isinstance(selectors, list):
selectors = [selectors]
for selector in selectors:
try:
if first_only:
element = soup.select_one(selector)
if element: return element
else:
elements = soup.select(selector)
if elements: return elements
except Exception as e:
continue
return None if first_only else []
def extract_issue_data(soup, main_url, config, domain):
issue_data = {}
current_year = datetime.now().strftime('%Y')
# --- Дата выпуска ---
meta_domains = ['d-governance.com.ua', 'lawjournal.com.ua', 'journal-knu.com.ua', 'devma.com.ua', 'ue-bulletin.com.ua', 'profedu.com.ua', 'arrcjournal.org', 'artnotes.com.ua']
if domain in meta_domains:
date_str = None
if domain == 'journal-knu.com.ua':
date_container = soup.select_one(config['selectors']['issue_date'])
if date_container:
full_text = safe_get_text(date_container)
date_match = re.search(r'(d{2}.d{2}.d{4})', full_text)
if date_match:
date_str = date_match.group(1)
elif domain == 'devma.com.ua':
date_container = soup.select_one(config['selectors']['issue_date'])
if date_container:
full_text = safe_get_text(date_container)
date_match = re.search(r'(d{2}.d{2}.d{4})', full_text)
if date_match:
date_str = date_match.group(1)
elif domain in ['ue-bulletin.com.ua', 'profedu.com.ua', 'arrcjournal.org', 'artnotes.com.ua']:
date_container = soup.select_one(config['selectors']['issue_date'])
if date_container:
full_text = safe_get_text(date_container)
date_match = re.search(r'(d{2}.d{2}.d{4})', full_text)
if date_match:
date_str = date_match.group(1)
else:
date_container = soup.select_one(config['selectors']['issue_date'])
if date_container:
full_text = safe_get_text(date_container)
date_match = re.search(r'(d{2}.d{2}.d{4})', full_text)
if date_match:
date_str = date_match.group(1)
year_match = re.search(r'(d{4})', main_url)
default_year = year_match.group(1) if year_match else current_year
issue_data['date'] = parse_date(date_str, config['patterns']['date_format'], default_year)
else:
date_container = find_element(soup, config['selectors']['issue_date_container'])
issue_date_str = safe_get_text(date_container) if date_container else None
url_year_part = main_url.split('-')[-1]
default_year = url_year_part if url_year_part.isdigit() else current_year
issue_data['date'] = parse_date(issue_date_str, config['patterns']['date_format'], default_year)
logging.info(f"Дата выпуска: {issue_data['date']}")
# --- Название журнала ---
journal_title = None
if domain == 'journal-knu.com.ua':
journal_title = "Journal of Kryvyi Rih National University"
desc_elem = soup.select_one('p.journal-description strong')
if desc_elem:
journal_title = safe_get_text(desc_elem, journal_title)
elif domain == 'devma.com.ua':
title_elem = soup.select_one(config['selectors']['journal_title'])
journal_title = safe_get_text(title_elem, "Development Management")
elif domain in ['ue-bulletin.com.ua', 'profedu.com.ua']:
title_elem = soup.select_one(config['selectors']['journal_title'])
journal_title = safe_get_text(title_elem, "University Economic Bulletin")
elif domain == 'arrcjournal.org':
title_elem = soup.select_one(config['selectors']['journal_title'])
journal_title = safe_get_text(title_elem, "Automobile Roads and Road Construction")
if journal_title == "Automobile Roads and Road Construction":
title_tag = soup.find('title')
if title_tag:
title_parts = title_tag.get_text(strip=True).split('|')
if title_parts:
journal_title = title_parts[0].strip()
elif domain == 'artnotes.com.ua':
title_elem = soup.select_one(config['selectors']['journal_title'])
journal_title = safe_get_text(title_elem, "Notes on Art Criticism")
elif domain in ['lawjournal.com.ua', 'd-governance.com.ua']:
journal_title = safe_get_text(soup.select_one(config['selectors']['journal_title']), "Unknown Journal")
else:
title_tag = soup.find('title')
journal_title_raw = safe_get_text(title_tag, "Unknown Journal")
journal_title = journal_title_raw.split('|')[0].strip()
issue_data['journal_title'] = journal_title
# --- Сокращение журнала ---
if domain in meta_domains:
issue_data['journal_abbrev'] = config['patterns'].get('journal_abbrev', 'unknown')
else:
issue_data['journal_abbrev'] = ''.join(word[0].lower() for word in journal_title.split() if word)
logging.info(f"Название журнала: {journal_title}, Сокращение: {issue_data['journal_abbrev']}")
# --- ISSN и e-ISSN ---
if domain in ['lawjournal.com.ua', 'journal-knu.com.ua', 'devma.com.ua', 'ue-bulletin.com.ua', 'profedu.com.ua', 'arrcjournal.org', 'artnotes.com.ua']:
issue_data['issn'] = config['patterns'].get('issn_print', 'Unknown ISSN')
issue_data['eissn'] = config['patterns'].get('eissn', 'Unknown e-ISSN')
elif domain == 'd-governance.com.ua':
issn, eissn = "Unknown ISSN", "Unknown e-ISSN"
footer_blocks = soup.select(config['selectors']['issn_footer'])
for block in footer_blocks:
text = safe_get_text(block)
if 'e-ISSN' in text:
match = re.search(r'e-ISSNs*([d-]+)', text)
if match: eissn = match.group(1)
elif 'ISSN' in text and 'e-ISSN' not in text:
match = re.search(r'ISSNs*([d-]+)', text)
if match: issn = match.group(1)
issue_data['issn'], issue_data['eissn'] = issn, eissn
else:
issn, eissn = "Unknown ISSN", "Unknown e-ISSN"
issn_container = find_element(soup, config['selectors']['issn_container'])
if issn_container:
text = issn_container.text.strip().lower()
issn_match = re.search(r'issns*([d-]+)', text)
eissn_match = re.search(r'e-issns*([d-]+)', text)
if issn_match: issn = issn_match.group(1)
if eissn_match: eissn = eissn_match.group(1)
issue_data['issn'], issue_data['eissn'] = issn, eissn
logging.info(f"Найдены: ISSN='{issue_data['issn']}', e-ISSN='{issue_data['eissn']}'")
# --- Том, номер, год ---
volume, issue, year = "unknown", "unknown", issue_data['date']['year']
if domain in meta_domains:
issue_title = safe_get_text(soup.select_one(config['selectors']['issue_title']))
match = re.search(config['patterns']['volume_issue_regex'], issue_title)
if match:
if domain == 'arrcjournal.org':
issue = match.group(1)
year = match.group(2)
volume = "1"
elif domain == 'artnotes.com.ua':
volume = match.group(1)
issue = match.group(2)
year = match.group(3)
else:
volume, issue = match.group(1), match.group(2)
if domain in ['ue-bulletin.com.ua', 'profedu.com.ua']:
year_match = re.search(r'(d{4})', main_url)
if year_match:
year = year_match.group(1)
else:
url_parts = main_url.strip('/').split('/')
volume_issue_part_indicator = config['patterns']['volume_issue_url_part']
volume_issue_url_part = next((part for part in url_parts if volume_issue_part_indicator in part), None)
if volume_issue_url_part:
parts = volume_issue_url_part.replace(volume_issue_part_indicator, '').split('-')
if len(parts) >= 2:
volume, issue = parts[0], parts[1]
issue_data['volume'], issue_data['issue'], issue_data['year'] = volume, issue, year
logging.info(f"Том: {volume}, Выпуск: {issue}, Год: {year}")
# --- DOI Выпуска ---
doi_prefix = config['patterns']['default_doi_prefix']
if domain == 'arrcjournal.org':
issue_data['doi'] = f"{doi_prefix}/arrc/{issue}.{year}"
elif domain == 'artnotes.com.ua':
issue_data['doi'] = f"{doi_prefix}/noac/{issue}.{year}"
else:
issue_data['doi'] = f"{doi_prefix}/ueb/{issue}.{year}"
if domain == 'journal-knu.com.ua':
doi_elem = soup.select_one(config['selectors'].get('issue_doi'))
if doi_elem:
raw_doi = safe_get_text(doi_elem)
if 'doi.org/' in raw_doi:
issue_data['doi'] = raw_doi.split('doi.org/')[-1]
else:
issue_data['doi'] = raw_doi
elif domain not in ['devma.com.ua', 'ue-bulletin.com.ua', 'profedu.com.ua', 'arrcjournal.org', 'artnotes.com.ua']:
issue_data['doi'] = f"{doi_prefix}-{year}-{issue}-{volume}"
issue_data['resource'] = main_url
logging.info(f"DOI выпуска: {issue_data['doi']}")
return issue_data
def extract_article_links_and_dois(soup, main_url, config, domain):
"""Извлекает ссылки и DOI статей со страницы выпуска."""
article_elements = soup.select(config['selectors']['article_item'])
article_links = []
article_dois = []
base_url = config.get('base_url') or f"{urlparse(main_url).scheme}://{urlparse(main_url).netloc}"
logging.info(f"Найдено {len(article_elements)} блоков статей.")
for article_el in article_elements:
link_elem = article_el.select_one(config['selectors']['article_link'])
if link_elem and link_elem.get('href'):
full_link = urljoin(base_url, link_elem['href'])
article_links.append(full_link)
else:
article_links.append(None)
logging.warning("Не найдена ссылка для статьи в блоке.")
doi_elem = article_el.select_one(config['selectors'].get('article_doi', ''))
if doi_elem:
doi_text = safe_get_text(doi_elem)
doi_match = re.search(r'(10.d+/S+)', doi_text)
if doi_match:
article_dois.append(doi_match.group(1))
else:
article_dois.append(None)
else:
article_dois.append(None)
valid_links = []
valid_dois = []
for i, link in enumerate(article_links):
if link:
valid_links.append(link)
valid_dois.append(article_dois[i] if i < len(article_dois) else None)
logging.info(f"Извлечено уникальных ссылок на статьи: {len(valid_links)}")
return valid_links, valid_dois
def extract_article_details(page, article_url, config, domain):
"""Извлекает данные о статье."""
article_data = {}
try:
logging.info(f"Открываем статью: {article_url}")
page.goto(article_url, timeout=60000)
page.wait_for_selector(config['selectors']['article_details_waitFor'], timeout=30000)
page.wait_for_timeout(1000)
soup = BeautifulSoup(page.content(), 'html.parser')
# --- ДЛЯ САЙТОВ, ИСПОЛЬЗУЮЩИХ МЕТА-ТЕГИ ---
if domain in ['ue-bulletin.com.ua', 'profedu.com.ua', 'arrcjournal.org', 'artnotes.com.ua']:
# Заголовок
title_meta = soup.find('meta', {'name': 'citation_title'})
article_data['title'] = title_meta['content'] if title_meta else "Title not found"
# Авторы
authors = []
for meta_author in soup.find_all('meta', {'name': 'citation_author'}):
author_name = meta_author['content'].strip()
if author_name:
authors.append(author_name)
article_data['authors'] = authors
logging.info(f"Авторы ({len(authors)}): {authors}")
# Аннотация
abstract = "Abstract not found"
desc_meta = soup.find('meta', {'name': 'DC.Description'})
if desc_meta and desc_meta.get('content'):
abstract = desc_meta['content'].strip()
article_data['abstract'] = abstract
# Даты (для arrcjournal формат DD.MM.YYYY, для остальных YYYY-MM-DD)
received_meta = soup.find('meta', {'name': 'DC.Date.dateSubmitted'})
accepted_meta = soup.find('meta', {'name': 'DC.Date.issued'})
received_date_str = received_meta['content'] if received_meta else None
accepted_date_str = accepted_meta['content'] if accepted_meta else None
default_year = datetime.now().year
if domain == 'arrcjournal.org':
article_data['print_date'] = parse_date(received_date_str, '%d.%m.%Y', default_year)
article_data['online_date'] = parse_date(accepted_date_str, '%d.%m.%Y', default_year)
else:
article_data['print_date'] = parse_date(received_date_str, '%Y-%m-%d', default_year)
article_data['online_date'] = parse_date(accepted_date_str, '%Y-%m-%d', default_year)
logging.info(f"Print дата: {article_data['print_date']}, Online дата: {article_data['online_date']}")
# Страницы
first_page_meta = soup.find('meta', {'name': 'citation_firstpage'})
last_page_meta = soup.find('meta', {'name': 'citation_lastpage'})
article_data['first_page'] = first_page_meta['content'] if first_page_meta else 'n/a'
article_data['last_page'] = last_page_meta['content'] if last_page_meta else 'n/a'
logging.info(f"Страницы: {article_data['first_page']}-{article_data['last_page']}")
# DOI
doi_meta = soup.find('meta', {'name': 'DC.Identifier.DOI'})
if doi_meta and doi_meta.get('content'):
doi_value = doi_meta['content']
if 'doi.org/' in doi_value:
doi_value = doi_value.split('doi.org/')[-1]
article_data['doi'] = doi_value
else:
article_data['doi'] = None
logging.info(f"DOI статьи: {article_data.get('doi')}")
# PDF ссылка
pdf_meta = soup.find('meta', {'name': 'citation_pdf_url'})
if pdf_meta and pdf_meta.get('content'):
article_data['pdf_link_relative'] = pdf_meta['content']
else:
article_data['pdf_link_relative'] = None
logging.info(f"PDF ссылка: {article_data.get('pdf_link_relative')}")
elif domain in ['devma.com.ua', 'd-governance.com.ua', 'lawjournal.com.ua']:
# Используем мета-теги для этих сайтов
title_meta = soup.find('meta', {'name': 'citation_title'})
article_data['title'] = title_meta['content'] if title_meta else "Title not found"
authors = []
for meta_author in soup.find_all('meta', {'name': 'citation_author'}):
author_name = meta_author['content'].strip()
if author_name:
authors.append(author_name)
article_data['authors'] = authors
logging.info(f"Авторы ({len(authors)}): {authors}")
abstract = "Abstract not found"
desc_meta = soup.find('meta', {'name': 'DC.Description'})
if desc_meta and desc_meta.get('content'):
abstract = desc_meta['content'].strip()
article_data['abstract'] = abstract
received_meta = soup.find('meta', {'name': 'DC.Date.dateSubmitted'})
accepted_meta = soup.find('meta', {'name': 'DC.Date.issued'})
received_date_str = received_meta['content'] if received_meta else None
accepted_date_str = accepted_meta['content'] if accepted_meta else None
default_year = datetime.now().year
article_data['print_date'] = parse_date(received_date_str, '%Y-%m-%d', default_year)
article_data['online_date'] = parse_date(accepted_date_str, '%Y-%m-%d', default_year)
logging.info(f"Print дата: {article_data['print_date']}, Online дата: {article_data['online_date']}")
first_page_meta = soup.find('meta', {'name': 'citation_firstpage'})
last_page_meta = soup.find('meta', {'name': 'citation_lastpage'})
article_data['first_page'] = first_page_meta['content'] if first_page_meta else 'n/a'
article_data['last_page'] = last_page_meta['content'] if last_page_meta else 'n/a'
logging.info(f"Страницы: {article_data['first_page']}-{article_data['last_page']}")
doi_meta = soup.find('meta', {'name': 'DC.Identifier.DOI'})
if doi_meta and doi_meta.get('content'):
doi_value = doi_meta['content']
if 'doi.org/' in doi_value:
doi_value = doi_value.split('doi.org/')[-1]
article_data['doi'] = doi_value
else:
article_data['doi'] = None
logging.info(f"DOI статьи: {article_data.get('doi')}")
pdf_meta = soup.find('meta', {'name': 'citation_pdf_url'})
article_data['pdf_link_relative'] = pdf_meta['content'] if pdf_meta else None
logging.info(f"PDF ссылка: {article_data.get('pdf_link_relative')}")
else:
# Для journal-knu.com.ua или humstudios.com.ua используем старый парсер
logging.info("Используется стандартный парсер для других сайтов.")
article_data['title'] = safe_get_text(find_element(soup, config['selectors']['article_title']), "Title not found")
# Авторы
authors = []
if domain == 'journal-knu.com.ua':
author_links = soup.select('div.article-authors a.article-author')
for link in author_links:
author_name = safe_get_text(link)
if author_name:
authors.append(author_name)
else:
author_anchor = find_element(soup, config['selectors']['authors_container'])
if author_anchor:
author_tags = author_anchor.find_next_siblings(config['selectors']['author_link'])
authors = [safe_get_text(tag).replace(',', '') for tag in author_tags if safe_get_text(tag)]
article_data['authors'] = authors
logging.info(f"Авторы ({len(authors)}): {authors}")
# Аннотация
abstract = "Abstract not found"
if domain == 'journal-knu.com.ua':
abstract_block = soup.select_one('div.article-info-block')
if abstract_block:
for p in abstract_block.find_all('p'):
if 'Abstract' in p.text or 'Анотація' in p.text:
abstract = p.text.replace('Abstract', '').replace('Анотація', '').strip()
break
else:
iframe_anchor = find_element(soup, config['selectors']['abstract_container'])
if iframe_anchor:
abstract_p = iframe_anchor.find_next_sibling('p')
if abstract_p:
abstract = safe_get_text(abstract_p)
article_data['abstract'] = abstract
# Даты
received_date_str = accepted_date_str = None
if domain == 'journal-knu.com.ua':
date_block = soup.select_one('div.article-info-block.additional-block')
if date_block:
text = safe_get_text(date_block)
received_match = re.search(r'Received:?s*(d{2}.d{2}.d{4})', text)
accepted_match = re.search(r'Accepted:?s*(d{2}.d{2}.d{4})', text)
if received_match: received_date_str = received_match.group(1)
if accepted_match: accepted_date_str = accepted_match.group(1)
else:
revised_date_str, accepted_date_str, received_date_str = None, None, None
dates_container = find_element(soup, config['selectors']['dates_container'])
if dates_container:
date_text = safe_get_text(dates_container)
parts = date_text.replace(';', ',').split(',')
for part in parts:
if config['patterns']['revised_date_keyword'] in part:
revised_date_str = part.split(config['patterns']['revised_date_keyword'])[-1].strip()
if config['patterns']['accepted_date_keyword'] in part:
accepted_date_str = part.split(config['patterns']['accepted_date_keyword'])[-1].strip()
if config['patterns']['received_date_keyword'] in part:
received_date_str = part.split(config['patterns']['received_date_keyword'])[-1].strip()
default_year = datetime.now().year
article_data['print_date'] = parse_date(received_date_str, config['patterns']['date_format'], default_year)
article_data['online_date'] = parse_date(accepted_date_str, config['patterns']['date_format'], default_year)
logging.info(f"Print дата: {article_data['print_date']}, Online дата: {article_data['online_date']}")
# Страницы
first_page, last_page = "n/a", "n/a"
if domain == 'journal-knu.com.ua':
for p in soup.select('div.article-info-block p'):
if 'Pages' in p.text:
pages_match = re.search(r'Pages:?s*(d+)-(d+)', p.text)
if pages_match:
first_page, last_page = pages_match.group(1), pages_match.group(2)
break
else:
pages_elems = find_element(soup, config['selectors']['pages_container'], first_only=False)
for elem in pages_elems:
pages_text = safe_get_text(elem)
if '-' in pages_text and pages_text.replace('-', '').isdigit():
first_page, last_page = pages_text.split('-')[0].strip(), pages_text.split('-')[-1].strip()
break
article_data['first_page'], article_data['last_page'] = first_page, last_page
logging.info(f"Страницы: {first_page}-{last_page}")
# DOI
doi = None
if domain == 'journal-knu.com.ua':
for p in soup.select('div.article-info-block p'):
if 'DOI' in p.text:
doi_match = re.search(r'DOI:?s*(10.d+/S+)', p.text)
if doi_match:
doi = doi_match.group(1)
break
else:
doi_elem = find_element(soup, config['selectors']['article_doi_link'])
if doi_elem and doi_elem.get('href'):
doi = doi_elem['href'].replace('https://doi.org/', '')
article_data['doi'] = doi
logging.info(f"DOI статьи: {article_data.get('doi')}")
# PDF ссылка
pdf_link = None
if domain == 'journal-knu.com.ua':
pdf_link_elem = soup.select_one('a[href*="/web/uploads/pdf/"]')
if pdf_link_elem and pdf_link_elem.get('href'):
pdf_link = pdf_link_elem['href']
else:
iframe_elem = find_element(soup, config['selectors'].get('article_pdf_iframe_src'))
if iframe_elem and iframe_elem.get('src'):
pdf_link = iframe_elem['src']
article_data['pdf_link_relative'] = pdf_link
logging.info(f"PDF ссылка: {article_data.get('pdf_link_relative')}")
article_data['resource'] = article_url
return article_data
except Exception as e:
logging.error(f"Ошибка при парсинге статьи {article_url}: {e}", exc_info=True)
return None
def build_crossref_xml(issue_data, articles_data, config, email):
"""Строит XML для CrossRef на основе данных выпуска и статей."""
if not issue_data:
logging.error("Данные выпуска отсутствуют, невозможно создать XML.")
return None
root = ET.Element('doi_batch', {'version': '4.4.2', 'xmlns': 'http://www.crossref.org/schema/4.4.2',
'xmlns:jats': 'http://www.ncbi.nlm.nih.gov/JATS1',
'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
'xsi:schemaLocation': 'http://www.crossref.org/schema/4.4.2 http://www.crossref.org/schema/deposit/crossref4.4.2.xsd'})
head = ET.SubElement(root, 'head')
ET.SubElement(head,
'doi_batch_id').text = f"{issue_data.get('journal_abbrev', 'unknown')}_batch_{datetime.now().strftime('%Y%m%d%H%M')}"
ET.SubElement(head, 'timestamp').text = datetime.now().strftime('%Y%m%d%H%M%S')
depositor = ET.SubElement(head, 'depositor')
ET.SubElement(depositor, 'depositor_name').text = 'WEB DEPOSITOR'
ET.SubElement(depositor, 'email_address').text = email
ET.SubElement(head, 'registrant').text = 'WEB-FORM'
body = ET.SubElement(root, 'body')
journal = ET.SubElement(body, 'journal')
journal_metadata = ET.SubElement(journal, 'journal_metadata', {'language': config.get('lang', 'en')})
ET.SubElement(journal_metadata, 'full_title').text = sanitize_xml_string(
issue_data.get('journal_title', 'Unknown Journal'))
if issue_data.get('journal_abbrev', 'unknown') != 'unknown':
ET.SubElement(journal_metadata, 'abbrev_title').text = issue_data['journal_abbrev'].upper()
if issue_data.get('issn', 'Unknown ISSN') != "Unknown ISSN":
ET.SubElement(journal_metadata, 'issn', {'media_type': 'print'}).text = issue_data['issn']
if issue_data.get('eissn', 'Unknown e-ISSN') != "Unknown e-ISSN":
ET.SubElement(journal_metadata, 'issn', {'media_type': 'electronic'}).text = issue_data['eissn']
journal_issue = ET.SubElement(journal, 'journal_issue')
issue_date = issue_data.get('date', {'month': '01', 'day': '01', 'year': datetime.now().year})
pub_date_issue = ET.SubElement(journal_issue, 'publication_date', {'media_type': 'online'})
ET.SubElement(pub_date_issue, 'month').text = issue_date.get('month', '01')
ET.SubElement(pub_date_issue, 'day').text = issue_date.get('day', '01')
ET.SubElement(pub_date_issue, 'year').text = str(issue_date.get('year', datetime.now().year))
if issue_data.get('volume', 'unknown') != "unknown":
volume_elem = ET.SubElement(journal_issue, 'journal_volume')
ET.SubElement(volume_elem, 'volume').text = issue_data['volume']
if issue_data.get('issue', 'unknown') != "unknown":
ET.SubElement(journal_issue, 'issue').text = issue_data['issue']
doi_data_issue = ET.SubElement(journal_issue, 'doi_data')
ET.SubElement(doi_data_issue, 'doi').text = issue_data.get('doi', 'unknown/issue_doi')
ET.SubElement(doi_data_issue, 'resource').text = issue_data.get('resource', '')
for data in articles_data:
if not data or not data.get('title') or data['title'] == "Title not found":
continue
article = ET.SubElement(journal, 'journal_article', {'publication_type': 'full_text'})
titles = ET.SubElement(article, 'titles')
ET.SubElement(titles, 'title').text = sanitize_xml_string(data['title'])
if data.get('authors'):
contributors = ET.SubElement(article, 'contributors')
for i, author_name in enumerate(data['authors']):
name_parts = author_name.split()
if name_parts:
person = ET.SubElement(contributors, 'person_name', {'contributor_role': 'author',
'sequence': 'first' if i == 0 else 'additional'})
given_name = name_parts[0]
surname = ' '.join(name_parts[1:]) if len(name_parts) > 1 else "Unknown"
ET.SubElement(person, 'given_name').text = sanitize_xml_string(given_name)
ET.SubElement(person, 'surname').text = sanitize_xml_string(surname)
abstract_text = data.get('abstract')
if abstract_text and abstract_text != "Abstract not found":
abstract_elem = ET.SubElement(article, '{http://www.ncbi.nlm.nih.gov/JATS1}abstract',
{'xml:lang': config.get('lang', 'en')})
ET.SubElement(abstract_elem, '{http://www.ncbi.nlm.nih.gov/JATS1}p').text = sanitize_xml_string(
abstract_text)
print_date = data.get('print_date', {'month': '01', 'day': '01', 'year': datetime.now().year})
online_date = data.get('online_date', {'month': '01', 'day': '01', 'year': datetime.now().year})
pub_date_print = ET.SubElement(article, 'publication_date', {'media_type': 'print'})
ET.SubElement(pub_date_print, 'month').text = print_date.get('month', '01')
ET.SubElement(pub_date_print, 'day').text = print_date.get('day', '01')
ET.SubElement(pub_date_print, 'year').text = str(print_date.get('year', datetime.now().year))
pub_date_online = ET.SubElement(article, 'publication_date', {'media_type': 'online'})
ET.SubElement(pub_date_online, 'month').text = online_date.get('month', '01')
ET.SubElement(pub_date_online, 'day').text = online_date.get('day', '01')
ET.SubElement(pub_date_online, 'year').text = str(online_date.get('year', datetime.now().year))
first_page, last_page = data.get('first_page'), data.get('last_page')
if first_page and first_page != 'n/a':
pages = ET.SubElement(article, 'pages')
ET.SubElement(pages, 'first_page').text = first_page
if last_page and last_page != 'n/a':
ET.SubElement(pages, 'last_page').text = last_page
doi_data_article = ET.SubElement(article, 'doi_data')
article_doi = data.get('doi')
if article_doi:
ET.SubElement(doi_data_article, 'doi').text = article_doi.replace('doi.org/', '')
ET.SubElement(doi_data_article, 'resource').text = data.get('resource', '')
if data.get('pdf_link_relative'):
pdf_url = data['pdf_link_relative']
if not pdf_url.startswith('http'):
base_url = config.get(
'base_url') or f"{urlparse(issue_data.get('resource', '')).scheme}://{urlparse(issue_data.get('resource', '')).netloc}"
pdf_url = urljoin(base_url, pdf_url)
collection = ET.SubElement(doi_data_article, 'collection', {'property': 'crawler-based'})
item = ET.SubElement(collection, 'item', {'crawler': 'iParadigms'})
ET.SubElement(item, 'resource').text = pdf_url
logging.info(f"Добавлена ссылка на PDF в XML: {pdf_url}")
try:
xml_str = ET.tostring(root, encoding='utf-8', method='xml')
dom = xml.dom.minidom.parseString(xml_str)
pretty_xml = dom.toprettyxml(indent=" ")
pretty_xml = pretty_xml.replace('ns0:', 'jats:')
return pretty_xml
except Exception as xml_format_error:
logging.error(f"Ошибка при форматировании XML: {xml_format_error}")
return ET.tostring(root, encoding='unicode')
def main():
depositor_email = ""
while not depositor_email:
depositor_email = input("Введите email депозитора и нажмите Enter: ").strip()
main_url = "https://artnotes.com.ua/en/journals/tom-26-1-2026"
output_file = r"C:UsersandreOneDriveDesktopcrossref_output_artnotes.xml"
run_headless = True
# --- УКАЗЫВАЕМ КОНКРЕТНУЮ СТАТЬЮ ---
# Вариант 1: по DOI (самый надежный)
target_doi = "10.63009/noac/1.2026.01"
# Вариант 2: по части заголовка (если DOI неизвестен)
# target_title_keyword = "Virtual exhibition formats"
# ------------------------------------
logging.info(f"Запускаем парсер для URL: {main_url}")
try:
config = get_config_for_url(main_url)
domain = urlparse(main_url).netloc.replace('www.', '')
except ValueError as e:
return
all_article_data, issue_data = [], None
with sync_playwright() as p:
browser = p.chromium.launch(headless=run_headless)
try:
page = browser.new_page()
page.goto(main_url, timeout=60000)
page.wait_for_selector(config['selectors']['issue_container_waitFor'], timeout=30000)
page.wait_for_timeout(1500)
main_soup = BeautifulSoup(page.content(), 'html.parser')
issue_data = extract_issue_data(main_soup, main_url, config, domain)
article_links, article_dois = extract_article_links_and_dois(main_soup, main_url, config, domain)
# --- ФИЛЬТРУЕМ СТАТЬИ, ОСТАВЛЯЕМ ТОЛЬКО НУЖНУЮ ---
target_article_url = None
for i, article_url in enumerate(article_links):
# Проверяем по DOI
if i < len(article_dois) and article_dois[i] == target_doi:
target_article_url = article_url
logging.info(f"Найдена целевая статья по DOI: {article_url}")
break
# Если не нашли по DOI, пробуем найти по заголовку
if not target_article_url:
logging.info("Поиск статьи по DOI не дал результата, пробуем по заголовку...")
# Перезагружаем страницу, чтобы получить свежий soup
page.goto(main_url, timeout=60000)
page.wait_for_selector(config['selectors']['issue_container_waitFor'], timeout=30000)
main_soup = BeautifulSoup(page.content(), 'html.parser')
article_elements = main_soup.select(config['selectors']['article_item'])
for article_el in article_elements:
link_elem = article_el.select_one(config['selectors']['article_link'])
if link_elem and link_elem.get('href'):
title = link_elem.get_text(strip=True)
if 'Virtual exhibition formats' in title:
target_article_url = urljoin(config.get('base_url'), link_elem['href'])
logging.info(f"Найдена целевая статья по заголовку: {target_article_url}")
break
# Если статья найдена, парсим её
if target_article_url:
article_details = extract_article_details(page, target_article_url, config, domain)
if article_details:
all_article_data.append(article_details)
logging.info(f"Успешно обработана статья: {article_details.get('title', 'Без заголовка')}")
else:
logging.error("Целевая статья не найдена на странице выпуска!")
except Exception as e:
logging.error(f"Критическая ошибка: {e}", exc_info=True)
finally:
browser.close()
logging.info(f"Собрано данных по {len(all_article_data)} статьям.")
if issue_data and all_article_data:
try:
pretty_xml = build_crossref_xml(issue_data, all_article_data, config, depositor_email)
if pretty_xml:
with open(output_file, 'w', encoding='utf-8') as f:
f.write(pretty_xml)
logging.info(f"XML файл успешно создан: {output_file}")
except Exception as e:
logging.error(f"Ошибка при записи XML файла: {e}", exc_info=True)
if __name__ == "__main__":
main()
