Personalized Open-Vocabulary Image Retrieval via Semantic and SRSiM-Based Social Features

Rim Fakhfakh, Ghada Feki, Chokri Ben Amar

Abstract


The large scale of image databases opens up new challenges for storing, organizing, and searching for data. The common challenge of image retrieval systems is to emphasize object detection and recognition from images to bridge the semantic gap between the Open-Vocabulary and Visual Content. This paper proposes a framework for a personalized image retrieval system which selects the most relevant images according to the specified query, the user interest, and the semantic interpretation. Our framework is composed of three major modules: (1) Users processing using Movielens as a social database. (2) Query processing and personalization. (3) Images processing by semantic annotation and clustering based on a new semantic similarity measure named SRSiM (Statistical Retina Similarity Measure) using statistical and knowledge-based approaches. The experimental study is performed on the NUS-WIDE dataset, a list of queries covering several social subjects and various user profiles extracted from the MovieLens database. The evaluation of the overall system shows promising results going up to 0.675 in terms of MAP (Mean Average Precision) measurement, which overcomes the score of other works established in the same conditions and using the same databases. The SRSiM measurement also reaches an accuracy of 77.77\% compared to Wordnet, Wikipedia, and Retina-based measurements

Full Text:

PDF

References


@article{xiong2016socializing,

author = {Li, Xirong and Uricchio, Tiberio and Ballan, Lamberto and Bertini, Marco and Snoek, Cees and Del Bimbo, Alberto},

year = {2016},

month = {06},

pages = {14:1-14:39},

title = {Socializing the Semantic Gap: A Comparative Survey on Image Tag Assignment, Refinement and Retrieval},

volume = {49},

journal = {ACM Computing Surveys},

doi = {10.1145/2906152}

}

@inproceedings{xiong2013classifying,

author = {Li, Xirong and Snoek, Cees},

year = {2013},

month = {10},

pages = {},

title = {Classifying Tag Relevance with Relevant Positive and Negative Examples},

journal = {MM 2013 - Proceedings of the 2013 ACM Multimedia Conference},

doi = {10.1145/2502081.2502129}

}

@ARTICLE{zhu2012Sampling,

author={Zhu, Shiai and Ngo, Chong-Wah and Jiang, Yu-Gang},

journal={IEEE Transactions on Multimedia},

title={Sampling and Ontologically Pooling Web Images for Visual Concept Learning},

year={2012},

volume={14},

number={4},

pages={1068-1078},

doi={10.1109/TMM.2012.2190387}

}

@article{bertier2009toward,

author = {Bertier, Marin and Guerraoui, Rachid and Kermarrec, Anne-Marie and Leroy, Vincent},

year = {2009},

month = {03},

pages = {},

title = {Toward Personalized Query Expansion},

doi = {10.1145/1578002.1578004}

}

@article{pitkow2002contextual,

added-at = {2018-03-19T12:24:51.000+0100},

address = {New York, NY, USA},

author = {Pitkow, James and Sch"{u}tze, Hinrich and Cass, Todd and Cooley, Rob and Turnbull, Don and Edmonds, Andy and Adar, Eytan and Breuel, Thomas},

doi = {10.1145/567498.567526},

issn = {0001-0782},

journal = {Commun. ACM},

keywords = {adaptive-search},

month = sep,

number = 9,

pages = {50--55},

posted-at = {2006-06-06 00:54:26},

priority = {2},

publisher = {ACM},

timestamp = {2018-03-19T12:24:51.000+0100},

title = {{Personalized search}},

volume = 45,

year = 2002

}

@article{Biancalana2013social,

author = {Biancalana, Claudio and Gasparetti, Fabio and Micarelli, Alessandro and Sansonetti, Giuseppe},

title = {Social semantic query expansion},

year = {2013},

issue_date = {September 2013},

publisher = {Association for Computing Machinery},

address = {New York, NY, USA},

volume = {4},

number = {4},

issn = {2157-6904},

doi = {10.1145/2508037.2508041},

journal = {ACM Trans. Intell. Syst. Technol.},

month = {oct},

articleno = {60},

numpages = {43},

}

@inproceedings{Bouadjenek2011personalized,

author = {Bouadjenek, Mohamed Reda and Hacid, Hakim and Bouzeghoub, Mokrane and Daigremont, Johann},

year = {2011},

month = {07},

pages = {1113-1114},

title = {Personnalized Social Query Expansion Using Social Bookmarking Systems},

doi = {10.1145/2009916.2010075}

}

@inproceedings{Bouadjenek2013sopra,

author = {Bouadjenek, Mohamed Reda and Hacid, Hakim and Bouzeghoub, Mokrane},

year = {2013},

month = {07},

pages = {861-864},

title = {SoPRa: A new social personalized ranking function for improving Web search},

isbn = {9781450320344},

doi = {10.1145/2484028.2484131}

}

@inproceedings{bao2007optimizing,

author = {Bao, Shenghua and Xue, Gui-Rong and Wu, Xiaoyuan and Yu, Yong and Fei, Ben and Su, Zhong},

year = {2007},

month = {05},

pages = {501-510},

title = {Optimizing Web Search Using Social Annotations},

journal = {16th International World Wide Web Conference, WWW2007},

doi = {10.1145/1242572.1242640}

}

@INPROCEEDINGS{aggoune2014personalized,

author={Aggoune, Aicha and Bouramoul, Ablelkrim and Kholladi, Mohamed-Khiereddine},

booktitle={2014 4th International Symposium ISKO-Maghreb: Concepts and Tools for knowledge Management (ISKO-Maghreb)},

title={Personalized indexing for heterogeneous multimedia data},

year={2014},

volume={},

number={},

pages={1-7},

doi={10.1109/ISKO-Maghreb.2014.7033460}

}

@article{safaris2015buildig,

author = {Sarafis, Ioannis and Diou, Christos and Delopoulos, Anastasios},

year = {2015},

month = {06},

pages = {},

title = {Building effective SVM concept detectors from clickthrough data for large-scale image retrieval},

volume = {4},

journal = {International Journal of Multimedia Information Retrieval},

doi = {10.1007/s13735-015-0080-5}

}

@article{Krizhevsky2012Imagenet,

author = {Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey},

year = {2012},

month = {01},

pages = {},

title = {ImageNet Classification with Deep Convolutional Neural Networks},

volume = {25},

journal = {Neural Information Processing Systems},

doi = {10.1145/3065386}

}

@article{rivas2014study,

author = {Rivas, Andreia and Iglesias, Eva and Borrajo, María},

year = {2014},

month = {03},

pages = {1-10},

title = {Study of Query Expansion Techniques and Their Application in the Biomedical Information Retrieval},

volume = {2014},

journal = {The Scientific World Journal},

doi = {10.1155/2014/132158}

}

@article{ghali2017context,

author = {El Ghali, Btihal and El Qadi, Abderrahim},

year = {2017},

month = {03},

pages = {},

title = {Context-aware query expansion method using Language Models and Latent Semantic Analyses},

volume = {50},

journal = {Knowledge and Information Systems},

doi = {10.1007/s10115-016-0952-x}

}

@article{witten2010effective,

author = {Witten, Ian and Milne, David},

year = {2010},

month = {06},

pages = {25-30},

title = {An Effective, Low-Cost Measure of Semantic Relatedness Obtained from Wikipedia Links},

publisher={Proceeding of AAAI Workshop on Wikipedia and Artificial Intelligence: An Evolving Synergy, AAAI Press, Chicago, USA}

}

@inproceedings{jira2014caffe,

author = {Jia, Yangqing and Shelhamer, Evan and Donahue, Jeff and Karayev, Sergey and Long, Jonathan and Girshick, Ross and Guadarrama, Sergio and Darrell, Trevor},

title = {Caffe: Convolutional Architecture for Fast Feature Embedding},

year = {2014},

isbn = {9781450330633},

publisher = {Association for Computing Machinery},

address = {New York, NY, USA},

doi = {10.1145/2647868.2654889},

booktitle = {Proceedings of the 22nd ACM International Conference on Multimedia},

pages = {675–678},

numpages = {4},

keywords = {parallel computation, open source, neural networks, machine learning, computer vision},

location = {Orlando, Florida, USA},

series = {MM '14}

}

@INPROCEEDINGS{fakhfakh2016personalizing,

author={Fakhfakh, Rim and Feki, Ghada and Ben Ammar, Anis and Ben Amar, Chokri},

booktitle={2016 IEEE International Conference on Systems, Man, and Cybernetics (SMC)},

title={Personalizing information retrieval: A new model for user preferences elicitation},

year={2016},

volume={},

number={},

pages={002091-002096},

doi={10.1109/SMC.2016.7844548}

}

@inproceedings{fakhfakh2014fuzzy,

author = {Fakhfakh, Rim and Ben Ammar, Anis and Ben Amar, Chokri},

title = {Fuzzy User Profile Modeling for Information Retrieval},

year = {2014},

isbn = {9789897580482},

publisher = {SCITEPRESS - Science and Technology Publications, Lda},

address = {Setubal, PRT},

doi = {10.5220/0005156304310436},

booktitle = {Proceedings of the International Joint Conference on Knowledge Discovery, Knowledge Engineering and Knowledge Management - Volume 1},

pages = {431–436},

numpages = {6},

keywords = {Social Network., Personalized Search, Fuzzy Logic},

location = {Rome, Italy},

series = {IC3K 2014}

}

@INPROCEEDINGS{fakhfakh2013enhancing,

author={Fakhfakh, Rim and Ksibi, Amel and Ben Ammar, Anis and Ben Amar, Chokri},

booktitle={2013 International Conference on Advanced Logistics and Transport},

title={Enhancing query interpretation by combining textual and visual analyses},

year={2013},

volume={},

number={},

pages={170-175},

doi={10.1109/ICAdLT.2013.6568454}

}

@Article{fakhfakh2012regimvid,

AUTHOR = {Fakhfakh, Rim and Ben Ammar, Anis and Ben Amar, Chokri},

TITLE = {REGIMvid at ImageCLEF 2012: Concept-based Query Refinement and Relevance-based Ranking Enhancement for Image Retrieval},

publisher = {CEUR-WS.org},

YEAR = {2014},

series = {{CEUR} Workshop Proceedings},

volume = {1178},

publisher = {CEUR-WS.org},

year = {2014},

url = {https://ceur-ws.org/Vol-1178},

}

@article{tekli2022combining,

author = {Tekli, Joe and Tekli, Gilbert and Chbeir, Richard},

year = {2022},

month = {01},

pages = {63-63},

title = {Combining offline and on-the-fly disambiguation to perform semantic-aware XML querying},

volume = {20},

journal = {Computer Science and Information Systems},

doi = {10.2298/CSIS220228063T}

}

@Article{feki2016regim2016,

AUTHOR = {Feki,G, Fakhfakh,R Bouhlel,N Ben Ammar and Ben Amar C},

TITLE = {REGIM@2016 Retrieving Diverse Social Images Task, in Proceedings of MediaEval},

publisher = {Mediaeval},

YEAR = {2016},

URL = {https://ceur-ws.org/Vol-1739/MediaEval_2016_paper_16.pdf},

}

@phdthesis{Peter2015,

author = "Peter Lofgren",

title = "efficient algorithms for personalized pagerank",

school = "Stanford University",

year = "2015"

}

@article{lee2017survey,

author = {Lee, Sanghoon and Masoud, Mohamed and Balaji, Janani and Belkasim, Saeid and Sunderraman, Rajshekhar and Moon, Seung-Jin},

year = {2017},

month = {06},

pages = {},

title = {A survey of tag-based information retrieval},

volume = {6},

journal = {International Journal of Multimedia Information Retrieval},

doi = {10.1007/s13735-016-0115-6}

}

@book{sang2014user,

author = {Sang, Jitao},

title = {User-centric Social Multimedia Computing},

year = {2014},

isbn = {3662446707},

publisher = {Springer Publishing Company, Incorporated},

}

@inproceedings{Musto2016learning,

author = {Musto, Cataldo and Semeraro, Giovanni and de Gemmis, Marco and Lops, Pasquale},

year = {2016},

month = {03},

pages = {729-734},

title = {Learning Word Embeddings from Wikipedia for Content-Based Recommender Systems},

volume = {9626},

isbn = {978-3-319-30670-4},

doi = {10.1007/978-3-319-30671-1_60}

}

@article{shen2021enhancing,

author = {Shen, Guojiang and Tan, Jiajia and Liu, Zhi and Kong, Xiangjie},

year = {2021},

month = {01},

pages = {64-64},

title = {Enhancing interactive graph representation learning for review-based item recommendation},

volume = {19},

journal = {Computer Science and Information Systems},

doi = {10.2298/CSIS210228064S}

}

@misc{Webber2015Semantic,

title={Semantic Folding Theory And its Application in Semantic Fingerprinting},

author={Francisco De Sousa Webber},

year={2016},

eprint={1511.08855},

archivePrefix={arXiv},

primaryClass={cs.AI}

}

@article{Elberrichi2008wordnet,

author = {Elberrichi, Zakaria and Rahmoun, Abdellatif and Bentaallah, Mohamed},

year = {2008},

month = {01},

pages = {16-24},

title = {Using WordNet for Text Categorization},

volume = {5},

journal = {Int. Arab J. Inf. Technol.}

}

@inbook{smyth2015collaboration,

author = {Smyth, Barry and Coyle, Maurice and Briggs, Peter and Mcnally, Kevin and O’Mahony, Michael},

year = {2015},

month = {01},

pages = {569-608},

title = {Collaboration, Reputation and Recommender Systems in Social Web Search},

isbn = {978-1-4899-7636-9},

doi = {10.1007/978-1-4899-7637-6_17}

}

@article{Tang2013social,

author = {Tang, Jiliang and Hu, Xia and Liu, Huan},

year = {2013},

month = {12},

pages = {1113-1133},

title = {Social recommendation: A review},

volume = {3},

journal = {Social Network Analysis and Mining},

doi = {10.1007/s13278-013-0141-9}

}

@Article{bok2022personalized,

AUTHOR = {Bok, Kyoungsoo and Song, Jinwoo and Lim, Jongtae and Yoo, Jaesoo},

TITLE = {Personalized Search Using User Preferences on Social Media},

JOURNAL = {Electronics},

VOLUME = {11},

YEAR = {2022},

NUMBER = {19},

ARTICLE-NUMBER = {3049},

ISSN = {2079-9292},

DOI = {10.3390/electronics11193049}

}

@Article{Jarrar2016coupled,

author={Jarrar, R. and Belkhatir, M.},

title={On the coupled use of signal and semantic concepts to bridge the semantic and user intention gaps for visual content retrieval},

journal={International Journal of Multimedia Information Retrieval},

year={2016},

month={Sep},

day={01},

volume={5},

number={3},

pages={165-172},

issn={2192-662X},

doi={10.1007/s13735-016-0101-z},

}

@article{zenebe2010user,

author = {Zenebe, A. and Zhou, Lina and Norcio, A.F.},

year = {2010},

month = {12},

pages = {3044-3063},

title = {User preferences discovery using fuzzy models},

volume = {161},

journal = {Fuzzy Sets and Systems},

doi = {10.1016/j.fss.2010.06.006}

}

@ARTICLE{Yang2017social,

author={Yang, Bo and Lei, Yu and Liu, Jiming and Li, Wenjie},

journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},

title={Social Collaborative Filtering by Trust},

year={2017},

volume={39},

number={8},

pages={1633-1647},

doi={10.1109/TPAMI.2016.2605085}

}

@article{yeh2021graph,

author = {Yeh, Jen-Yuan and Tsai, Cheng-Jung},

year = {2021},

month = {01},

pages = {42-42},

title = {A graph-based feature selection method for learning to rank using spectral clustering for redundancy minimization and biased PageRank for relevance analysis},

volume = {19},

journal = {Computer Science and Information Systems},

doi = {10.2298/CSIS201220042Y}

}

@INPROCEEDINGS{feki2015knowledge,

author={Feki, Ghada and Fakhfakh, Rim and Ben Ammar, Anis and Ben Amar, Chokri},

booktitle={2015 15th International Conference on Intelligent Systems Design and Applications (ISDA)},

title={Knowledge structures: Which one to use for the query disambiguation?},

year={2015},

volume={},

number={},

pages={499-504},

doi={10.1109/ISDA.2015.7489166}

}

@article{zhang2017index,

author = {Zhang, Mingxi and Li, Pohan and Wang, Wei},

year = {2017},

month = {05},

pages = {},

title = {An index-based algorithm for fast on-line query processing of latent semantic analysis},

volume = {12},

journal = {PLoS ONE},

doi = {10.1371/journal.pone.0177523}

}

@Article{zhou2023shorttext,

AUTHOR = {Zhou, Ya and Li, Cheng and Huang, Guimin and Guo, Qingkai and Li, Hui and Wei, Xiong},

TITLE = {A Short-Text Similarity Model Combining Semantic and Syntactic Information},

JOURNAL = {Electronics},

VOLUME = {12},

YEAR = {2023},

NUMBER = {14},

ARTICLE-NUMBER = {3126},

ISSN = {2079-9292},

DOI = {10.3390/electronics12143126}

}

@ARTICLE{Rewaria2021data,

title = "Data privacy in social media platform: Issues and challenges",

author = "Rewaria, Sakshi",

journal = "SSRN Electron. J.",

publisher = "Elsevier BV",

year = 2021,

language = "en"

}

@Article{ali2018privacy,

AUTHOR = {Ali, Shaukat and Islam, Naveed and Rauf, Azhar and Din, Ikram Ud and Guizani, Mohsen and Rodrigues, Joel J. P. C.},

TITLE = {Privacy and Security Issues in Online Social Networks},

JOURNAL = {Future Internet},

VOLUME = {10},

YEAR = {2018},

NUMBER = {12},

ARTICLE-NUMBER = {114},

URL = {https://www.mdpi.com/1999-5903/10/12/114},

ISSN = {1999-5903},

DOI = {10.3390/fi10120114}

}

@article{rossi2022challenges,

author = {Rossi, Arianna and Kocyigit, Emre and Arenas, Mónica and Hani, Moad},

year = {2022},

month = {08},

pages = {554-561},

title = {Challenges of protecting confidentiality in social media data and their ethical import},

doi = {10.1109/EuroS&PW55150.2022.00066}

}

@article{fakhfakh2017deep,

author = {Fakhfakh, Rim and Anis, Ben ammar and Ben Amar, Chokri},

year = {2017},

month = {01},

pages = {},

title = {Deep Learning-Based Recommendation: Current Issues and Challenges},

volume = {8},

journal = {International Journal of Advanced Computer Science and Applications},

doi = {10.14569/IJACSA.2017.081209}

}




DOI: https://doi.org/10.31449/inf.v49i9.5684

Creative Commons License
This work is licensed under a Creative Commons Attribution 3.0 License.