Team

@inproceedings{	20.500.12708_191183,

title = {Potentials of Combining Local Knowledge and LLMs for Recommender Systems},

author = {Thomas Elmar Kolb and Ahmadou Wagne and Mete Sertkan and Julia Neidhardt},

editor = {Vito Walter Anelli and Pierpaolo Basile and Gerard De Melo and Francesco Donini and Antonio Ferrara and Cataldo Musto and Fedelucio Narducci and Azzurra Ragone and Markus Zanker},

doi = {10.34726/5334},

year  = {2023},

date = {2023-01-01},

volume = {3560},

pages = {61–64},

publisher = {CEUR-WS.org},

series = {CEUR Workshop Proceedings},

abstract = {LLMs have revolutionized the understanding and generation of natural language, offering new possibilities for enhancing recommendation systems. In previous studies, LLMs exploit their global knowledge to provide zero- or few-shot recommendations. In this work, we aim to highlight the opportunities that LLMs pose to enrich the field of recommender systems combined with local knowledge. We propose to view recommender systems combined with LLMs from a broader perspective, recognizing them not merely as another method to replace existing recommendation approaches, but rather as a complementary and powerful approach to enhance and augment the overall recommendation process.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_191697,

title = {Exploring Effect-Size-Based Meta-Analysis for Multi-Dataset Evaluation},

author = {Mete Sertkan and Sophia Althammer and Sebastian Hofstätter and Peter Knees and Julia Neidhardt},

doi = {10.34726/5352},

year  = {2023},

date = {2023-01-01},

volume = {3476},

publisher = {CEUR-WS.org},

series = {CEUR Workshop Proceedings},

abstract = {In this paper, we address the essential yet complex task of evaluating Recommender Systems (RecSys) across multiple datasets. This is critical for gauging their overall performance and applicability in various contexts. Owing to the unique characteristics of each dataset and the variability in algorithm performance, we propose the adoption of effect-size-based meta-analysis, a proven tool in comparative research. This approach enables us to compare a “treatment model” and a “control model” across multiple datasets, offering a comprehensive evaluation of their performance. Through two case studies, we highlight the flexibility and effectiveness of this method in multi-dataset evaluations, irrespective of the metric utilized. The power of forest plots in providing an intuitive and concise summarization of our analysis is also demonstrated, which significantly aids in the communication of research findings. Our work provides valuable insights into leveraging these methodologies to draw more reliable and validated conclusions on the generalizability and robustness of RecSys models.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_192941,

title = {Ranger: A Toolkit for Effect-Size Based Multi-Task Evaluation},

author = {Mete Sertkan and Sophia Althammer and Sebastian Hofstätter},

editor = {Bollegala Danushka},

doi = {10.18653/v1/2023.acl-demo.56},

year  = {2023},

date = {2023-01-01},

volume = {Volume 3: System Demonstrations},

pages = {581–587},

publisher = {Association for Computational Linguistics},

abstract = {In this paper, we introduce Ranger - a toolkit to facilitate the easy use of effect-size-based meta-analysis for multi-task evaluation in NLP and IR. We observed that our communities often face the challenge of aggregating results over incomparable metrics and scenarios, which makes conclusions and take-away messages less reliable. With Ranger, we aim to address this issue by providing a task-agnostic toolkit that combines the effect of a treatment on multiple tasks into one statistical evaluation, allowing for comparison of metrics and computation of an overall summary effect. Our toolkit produces publication-ready forest plots that enable clear communication of evaluation results over multiple tasks. Our goal with the ready-to-use Ranger toolkit is to promote robust, effect-size-based evaluation and improve evaluation standards in the community. We provide two case studies for common IR and NLP settings to highlight Ranger’s benefits.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_175977,

title = {Diversifying Sentiments in News Recommendation},

author = {Mete Sertkan and Sophia Althammer and Sebastian Hofstätter and Julia Neidhardt},

doi = {10.34726/3903},

year  = {2022},

date = {2022-01-01},

volume = {3228},

series = {CEUR Workshop Proceedings},

abstract = {Personalized news recommender systems are widely deployed to filter the information overload caused by the sheer amount of news produced daily. Recommended news articles usually have a sentiment similar to the sentiment orientation of the previously consumed news, creating a self-reinforcing cycle of sentiment chambers around people. Wu et al. introduced SentiRec – a sentiment diversity-aware neural news recommendation model to counter this lack of diversity. 

 In this work, we reproduce SentiRec without access to the original source code and data sample. We re-implement SentiRec from scratch and use the Microsoft MIND dataset (same source but different subset as in the original work) for our experiments. We evaluate and discuss our reproduction from different perspectives. While the original paper mainly has a user-centric perspective on sentiment diversity by comparing the recommendation list to the user’s interaction history, we also analyze the intra-list sentiment diversity of the recommendation list. Additionally, we study the effect of sentiment diversification on topical diversity. Our results suggest that SentiRec does not generalize well to other data since the compared baselines already perform well, opposing the original work’s findings. While the original SentiRec utilizes a rule-based sentiment analyzer, we also study a pre-trained neural sentiment analyzer. However, we observe no improvements in effectiveness nor in sentiment diversity. To foster reproducibility, we make our source code publicly available.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_150340,

title = {The Role of Bias in News Recommendation in the Perception of the Covid-19 Pandemic},

author = {Thomas Elmar Kolb and Irina Nalis and Mete Sertkan and Julia Neidhardt},

editor = {Kolb Thomas},

doi = {10.48550/ARXIV.2209.07608},

year  = {2022},

date = {2022-01-01},

abstract = {News recommender systems (NRs) have been shown to shape public discourse and to enforce behaviors that have a critical, oftentimes detrimental effect on democracies. Earlier research on the impact of media bias has revealed their strong impact on opinions and preferences. Responsible NRs are supposed to have depolarizing capacities, once they go beyond accuracy measures. We performed sequence prediction by using the BERT4Rec algorithm to investigate the interplay of news of coverage and user behavior. Based on live data and training of a large data set from one news outlet "event bursts", "rally around the flag" effect and "filter bubbles" were investigated in our interdisciplinary approach between data science and psychology. Potentials for fair NRs that go beyond accuracy measures are outlined via training of the models with a large data set of articles, keywords, and user behavior. The development of the news coverage and user behavior of the COVID-19 pandemic from primarily medical to broader political content and debates was traced. Our study provides first insights for future development of responsible news recommendation that acknowledges user preferences while stimulating diversity and accountability instead of accuracy, only.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_153156,

title = {Exploring Expressed Emotions for Neural News Recommendation},

author = {Mete Sertkan and Julia Neidhardt},

doi = {10.1145/3511047.3536414},

year  = {2022},

date = {2022-01-01},

pages = {22–28},

publisher = {Association for Computing Machinery},

address = {New York, NY, United States},

abstract = {Due to domain-specific challenges such as short item lifetimes and continuous cold-start issues, news recommender systems rely more on content-based methods to deduce reliable user models and make personalized recommendations. Research has shown that alongside the content of an item, the way it is presented to the users also plays a critical role. In this work, we focus on the effect of incorporating expressed emotions within news articles on recommendation performance. We propose a neural news recommendation model that disentangles semantic and emotional modeling of news articles and users. While we exploit the textual content for the semantic representation, we extract and combine emotions of different information levels for the emotional representation. Offline experiments on a real-world dataset show that our approach outperforms non-emotion-aware solutions significantly. Finally, we provide a future outline, where we plan to investigate a) the online performance and b) the explainability/explorability of our approach.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_153166,

title = {Towards an Approach for Analyzing Dynamic Aspects of Bias and Beyond-Accuracy Measures},

author = {Julia Neidhardt and Mete Sertkan},

editor = {Ludovico Boratto and Stefano Faralli and Mirko Marras and stilo},

doi = {10.1007/978-3-031-09316-6_4},

year  = {2022},

date = {2022-01-01},

volume = {1610},

pages = {35–42},

publisher = {Springer Cham},

abstract = {The quality of recommender systems has traditionally only been assessed using accuracy measures. Research has shown that accuracy is only one side of the medallion and that we should also consider quality features that go beyond accuracy. Recently, also fairness-related aspects and bias have increasingly been considered as outcome dimensions in this context. While beyond-accuracy measures including diversity, novelty and serendipity and bias in recommendation have been subject to the research discourse, their interrelation and temporal and group dynamics are clearly under-explored. In this position paper, we propose an approach that groups users based on their behaviors and preferences and that addresses beyond-accuracy needs of those groups while controlling for bias. Further, we consider the analysis of long-term dynamics of different interrelated beyond-accuracy measures and bias as crucial research direction since it helps to advance the field and to address societal issues related to recommender systems and personalization.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_152296,

title = {Introducing Neural Bag of Whole-Words with ColBERTer: Contextualized Late Interactions using Enhanced Reduction},

author = {Sebastian Hofstätter and Omar Khattab and Sophia Althammer and Mete Sertkan and Allan Hanbury},

doi = {10.1145/3511808.3557367},

year  = {2022},

date = {2022-01-01},

pages = {737–747},

publisher = {Association for Computing Machinery (ACM)},

address = {New York, US},

abstract = {Recent progress in neural information retrieval has demonstrated large gains in quality, while often sacrificing efficiency and interpretability compared to classical approaches. We propose ColBERTer, a neural retrieval model using contextualized late interaction (ColBERT) with enhanced reduction. Along the effectiveness Pareto frontier, ColBERTer dramatically lowers ColBERT's storage requirements while simultaneously improving the interpretability of its token-matching scores. To this end, ColBERTer fuses single-vector retrieval, multi-vector refinement, and optional lexical matching components into one model. For its multi-vector component, ColBERTer reduces the number of stored vectors by learning unique whole-word representations and learning to identify and remove word representations that are not essential to effective scoring. We employ an explicit multi-task, multi-stage training to facilitate using very small vector dimensions. Results on the MS MARCO and TREC-DL collection show that ColBERTer reduces the storage footprint by up to 2.5x, while maintaining effectiveness. With just one dimension per token in its smallest setting, ColBERTer achieves index storage parity with the plaintext size, with very strong effectiveness results. Finally, we demonstrate ColBERTer's robustness on seven high-quality out-of-domain collections, yielding statistically significant gains over traditional retrieval baselines.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_193896,

title = {A Time-Optimized Content Creation Workflow for Remote Teaching},

author = {Sebastian Hofstätter and Sophia Althammer and Mete Sertkan and Allan Hanbury},

doi = {10.1145/3478431.3499421},

year  = {2022},

date = {2022-01-01},

volume = {1},

pages = {731–737},

publisher = {Association for Computing Machinery},

address = {New York, NY, United States},

abstract = {We describe our workflow to create an engaging remote learning experience for a university course, while minimizing the post-production time of the educators. We make use of ubiquitous and commonly free services and platforms, so that our workflow is inclusive for all educators and provides polished experiences for students. Our learning materials provide for each lecture: 1) a recorded video, uploaded on YouTube, with exact slide timestamp indices, which enables an enhanced navigation UI; and 2) a high-quality flow-text automated transcript of the narration with proper punctuation and capitalization, improved with a student participation workflow on GitHub. All these results could be created by hand in a time consuming and costly way. However, this would generally exceed the time available for creating course materials. Our main contribution is to automate the transformation and post-production between raw narrated slides and our published materials with a custom toolchain. Furthermore, we describe our complete workflow: from content creation to transformation and distribution. Our students gave us overwhelmingly positive feedback and especially liked our use of ubiquitous platforms. The most used feature was YouTube's chapter UI enabled through our automatically generated timestamps. The majority of students, who started using the transcripts, continued to do so. Every single transcript was corrected by students, with an average word-change of 6%. We conclude with the positive feedback that our enhanced content formats are much appreciated and utilized. Important for educators is how our low overhead production workflow was sustainable throughout a busy semester.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_137002,

title = {PARM: A Paragraph Aggregation Retrieval Model for Dense Document-to-Document Retrieval},

author = {Sophia Althammer and Sebastian Hofstätter and Mete Sertkan and Suzan Verberne and Allan Hanbury},

doi = {10.1007/978-3-030-99736-6_2},

year  = {2022},

date = {2022-01-01},

pages = {19–34},

publisher = {Springer},

abstract = {Dense passage retrieval (DPR) models show great effectiveness gains in first stage retrieval for the web domain. However in the web domain we are in a setting with large amounts of training data and a query-to-passage or a query-to-document retrieval task. We investigate in this paper dense document-to-document retrieval with limited labelled target data for training, in particular legal case retrieval. In order to use DPR models for document-to-document retrieval, we propose a Paragraph Aggregation Retrieval Model (PARM) which liberates DPR models from their limited input length. PARM retrieves documents on the paragraph-level: for each query paragraph, relevant documents are retrieved based on their paragraphs. Then the relevant results per query paragraph are aggregated into one ranked list for the whole query document. For the aggregation we propose vector-based aggregation with reciprocal rank fusion (VRRF) weighting, which combines the advantages of rank-based aggregation and topical aggregation based on the dense embeddings. Experimental results show that VRRF outperforms rank-based aggregation strategies for dense document-to-document retrieval with PARM. We compare PARM to document-level retrieval and demonstrate higher retrieval effectiveness of PARM for lexical and dense first-stage retrieval on two different legal case retrieval collections. We investigate how to train the dense retrieval model for PARM on limited target data with labels on the paragraph or the document-level. In addition, we analyze the differences of the retrieved results of lexical and dense retrieval with PARM.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_58755,

title = {Modeling Users and Items for Recommenders:There Is More than Semantics},

author = {Mete Sertkan},

doi = {10.1145/3460231.3473898},

year  = {2021},

date = {2021-01-01},

booktitle = {Fifteenth ACM Conference on Recommender Systems},

publisher = {ACM},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_58382,

title = {Eliciting Touristic Profiles: A User Study on Picture Collections},

author = {Mete Sertkan and Julia Neidhardt and Hannes Werthner},

editor = {Tsvi Kuflik and Ilaria Torre and Robin Burke and Cristina Gena},

doi = {10.1145/3340631.3394868},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 28th ACM Conference on User Modeling, Adaptation and Personalization},

publisher = {Association for Computing Machinery, New York, NY, USA},

abstract = {Eliciting the preferences and needs of tourists is challenging, since people often have difficulties to explicitly express them, especially in the initial phase of travel planning. Recommender systems employed at the early stage of planning can therefore be very beneficial to the general satisfaction of a user. Previous studies have explored pictures as a tool of communication and as a way to implicitly deduce a traveller's preferences and needs. In this paper, we conduct a user study to verify previous claims and conceptual work on the feasibility of modelling travel interests from a selection of a user's pictures. We utilize fine-tuned convolutional neural networks to compute a vector representation of a picture, where each dimension corresponds to a travel behavioural pattern from the traditional Seven-Factor model. In our study, we followed strict privacy principles and did not save uploaded pictures after computing their vector representation. We aggregate the representations of the pictures of a user into a single user representation, ie, touristic profile, using different strategies. In our user study with 81 participants, we let users adjust the predicted touristic profile and confirm the usefulness of our approach. Our results show that given a collection of pictures the touristic profile of a user can be determined.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_58381,

title = {From Pictures to Travel Characteristics: Deep Learning-Based Profiling of Tourists and Tourism Destinations},

author = {Mete Sertkan and Julia Neidhardt and Hannes Werthner},

editor = {Julia Neidhardt and Wolfgang Wörndl},

doi = {10.1007/978-3-030-36737-4_12},

isbn = {9783030367374},

year  = {2020},

date = {2020-01-01},

booktitle = {Information and Communication Technologies in Tourism 2020},

pages = {142–153},

publisher = {Springer},

abstract = {Tourism products are complex and strongly tied to emotions. Thus, it is not easy for consumers to explicitly communicate their travel preferences, needs, and interest, especially in the early phase of travel decision making. In the spirit of the idiom "A picture is worth a thousand words" we utilize pictures to characterize tourists as well as tourism destinations in order to build the foundations of a recommender system (RS). In this work all entities (i.e., users and items) are characterized using the Seven-Factor Model. Pre-labelled pictures are used in order to train convolutional neural networks (CNN) in a transfer learning manner with the goal to extract the Seven-Factors of a given picture. We demonstrate that touristic characteristics can be extracted out of pictures. Furthermore, we show that those characteristics can be aggregated for a collection of pictures, such that a representation of a user or a destination can be determined respectively.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_58385,

title = {PicTouRe - A Picture-Based Tourism Recommender},

author = {Mete Sertkan and Julia Neidhardt and Hannes Werthner},

doi = {10.1145/3383313.3411526},

year  = {2020},

date = {2020-01-01},

booktitle = {Fourteenth ACM Conference on Recommender Systems},

publisher = {Association for Computing Machinery, New York, United States},

abstract = {We present PicTouRe-a picture-based tourism recommender. PicTouRe aims to mitigate people´s difficulties in explicitly expressing their touristic preferences, which is even more challenging in the initial phase of travel decision making. Addressing this issue, with PicTouRe we follow the idiom "a picture is worth a thousand words" and use pictures as a tool to implicitly elicit peoples´ touristic preferences. We describe the core concept of PicTouRe-the Generic Profiler, which in essence determines an explainable vector representation, ie, touristic profile, given any picture collection as input. We showcase a user´s journey through PicTouRe and describe the steps behind. Finally, we present results of a first user study supporting our approach. PicTouRe is available under https://pictoprof. ec. tuwien. ac. at and a demo video under https://youtu. be/xZnXLPcenEs.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_58253,

title = {Fine-Grained Relevance Annotations for Multi-Task Document Ranking and Question Answering},

author = {Sebastian Hofstätter and Markus Zlabinger and Mete Sertkan and Michael Schröder and Allan Hanbury},

doi = {10.1145/3340531.3412878},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 29th ACM International Conference on Information & Knowledge Management},

publisher = {Association for Computing Machinery},

abstract = {There are many existing retrieval and question answering datasets. However, most of them either focus on ranked list evaluation or single-candidate question answering. This divide makes it challenging to properly evaluate approaches concerned with ranking documents and providing snippets or answers for a given query. In this work, we present FiRA: a novel dataset of Fine-Grained Relevance Annotations. We extend the ranked retrieval annotations of the Deep Learning track of TREC 2019 with passage and word level graded relevance annotations for all relevant documents. We use our newly created data to study the distribution of relevance in long documents, as well as the attention of annotators to specific positions of the text. As an example, we evaluate the recently introduced TKL document ranking model. We find that although TKL exhibits state-of-the-art retrieval results for long documents, it misses many relevant passages.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_58383,

title = {DEXA: Supporting Non-Expert Annotators with Dynamic Examples from Experts},

author = {Markus Zlabinger and Marta Sabou and Sebastian Hofstätter and Mete Sertkan and Allan Hanbury},

editor = {Jimmy Huang and Yi Chang},

doi = {10.1145/3397271.3401334},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval},

publisher = {Association for Computing Machinery, New York, NY, United States},

abstract = {The success of crowdsourcing based annotation of text corpora depends on ensuring that crowdworkers are sufficiently well-trained to perform the annotation task accurately. To that end, a frequent approach to train annotators is to provide instructions and a few example cases that demonstrate how the task should be performed (referred to as the CONTROL approach). These globally defined "task-level examples", however, (i) often only cover the common cases that are encountered during an annotation task; and (ii) require effort from crowdworkers during the annotation process to find the most relevant example for the currently annotated sample. To overcome these limitations, we propose to support workers in addition to task-level examples, also with "task-instance level" examples that are semantically similar to the currently annotated data sample (referred to as Dynamic Examples for Annotation, DEXA). Such dynamic examples can be retrieved from collections previously labeled by experts, which are usually available as gold standard dataset. We evaluate DEXA on a complex task of annotating participants, interventions, and outcomes (known as PIO) in sentences of medical studies. The dynamic examples are retrieved using BioSent2Vec, an unsupervised semantic sentence similarity method specific to the biomedical domain. Results show that (i) workers of the DEXA approach reach on average much higher agreements (Cohen's Kappa) to experts than workers of the the CONTROL approach (avg. of 0.68 to experts in DEXA vs. 0.40 in CONTROL); (ii) already three per majority voting aggregated annotations of the DEXA approach reach substantial agreements to experts of 0.78/0.75/0.69 for P/I/O (in CONTROL 0.73/0.58/0.46). Finally, (iii) we acquire explicit feedback from workers and show that in the majority of cases (avg. 72%) workers find the dynamic examples useful.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_58129,

title = {From Pictures to Touristic Profiles: A Deep-Learning Based Approach},

author = {Mete Sertkan and Julia Neidhardt and Hannes Werthner},

isbn = {978-1-5262-0820-0},

year  = {2019},

date = {2019-01-01},

booktitle = {ProceedingsDSRS-Turing´19.},

pages = {75–78},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_57685,

title = {Documents, Topics, and Authors: Text Mining of Online News},

author = {Mete Sertkan and Julia Neidhardt and Hannes Werthner},

doi = {10.1109/cbi.2019.00053},

year  = {2019},

date = {2019-01-01},

booktitle = {2019 IEEE 21st Conference on Business Informatics (CBI)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{	20.500.12708_57174,

title = {Mapping of Tourism Destinations to Travel Behavioural Patterns},

author = {Mete Sertkan and Julia Neidhardt and Hannes Werthner},

doi = {10.1007/978-3-319-72923-7_32},

isbn = {9783319729237},

year  = {2018},

date = {2018-01-01},

booktitle = {Information and Communication Technologies in Tourism 2018},

pages = {422–434},

publisher = {Springer, Cham},

abstract = {Tourism is an information intensive domain, where recommender systems have become an essential tool to guide customers to the right products. However, they are facing major challenges, since tourism products are considered as complex and emotional. It has been shown that the seven-factor model is a legitimate way to counter some of these challenges. However, in order to recommend an item, it has also to be described in terms of this model. This work´s aim is to find a scalable way to map tourism destinations, defined by their attributes, to the seven-factor model. Through statistical analysis and learning methods it is shown that there is a significant relationship between particular destination features and the seven-factors and that destinations can be grouped in a meaningful way using their attributes.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}