Real-time Semantic Healthcare System: Visual Risks Identification for Elders and Children

Malak Belkebir, Toufik Messaoud Maarouk, Brahim Nini

Abstract


Deep learning and data-driven approaches are commonly used to avoid accidents involving elders and children. However, existing models are limited by a semantic gap, hindering their ability to infer new risks that have not been previously trained. In this paper, a realtime healthcare system is developed to identify and infer visual risks in surveillance videos for elders and children. The system consists of three main modules: "visual information extraction," which leverages advancements in artificial vision techniques such as GCD, YOLO, ResNet18, and IoU; "ontology modeling," where a new high-level ontology named "Risks-Identification-Onto" is constructed based on FOL and DLs; and "risk identification," where the system infers risks by deducing new knowledge through the reasoning of generated formal rules over the data-driven techniques. Additionally, the system generates a high-level semantic description of the risky situation. Four common risk scenarios - "Hurt," "Burn," "Existing-in-dangerous-places," and "Hit"- are selected to evaluate the effectiveness of the proposed system. Evaluation is conducted using the Charades and A2D datasets, demonstrating the system's efficiency in identifying and inferring risks in realtime with an accuracy ranging from 97.61% to 99.43%.

Full Text:

PDF PDF

References


@article{shi2023association,

title={The association between wind speed and the risk of injuries among preschool children: New insight from a sentinel-surveillance-based study},

author={Shi, Congxing and Lin, Xiao and Huang, Tingyuan and Zhang, Kai and Liu, Yanan and Tian, Tian and Wang, Pengyu and Chen, Shimin and Guo, Tong and Li, Zhiqiang and others},

journal={Science of the total environment},

volume={856},

pages={159005},

year={2023},

publisher={Elsevier}

}

@article{dawson2024monitoring,

title={Monitoring falls in residential aged care facilities: Agreement between falls incident reports and progress notes},

author={Dawson, Rik and Feng, Annie and Oliveira, Juliana S and Hassett, Leanne and Sherrington, Catherine and Pinheiro, Marina B},

journal={Australasian journal on ageing},

year={2024},

publisher={Wiley Online Library}

}

@inproceedings{ohnuki2020visual,

title={A Visual Monitoring Method for Infants in a Room},

author={Ohnuki, Takumi and Abe, Toru and Suganuma, Takuo},

booktitle={2020 IEEE 9th Global Conference on Consumer Electronics (GCCE)},

pages={258--259},

year={2020},

organization={IEEE}

}

@inproceedings{wang2019learning,

title={A learning-based prediction model for baby accidents},

author={Wang, Peng-Jie and Lien, Shao-Fu and Lee, Ming-Sui},

booktitle={2019 IEEE International Conference on Image Processing (ICIP)},

pages={629--633},

year={2019},

organization={IEEE}

}

@article{doukari2024ontology,

title={An ontology-based tool for safety management in building renovation projects},

author={Doukari, Omar and Wakefield, James and Martinez, Pablo and Kassem, Mohamad},

journal={Journal of Building Engineering},

volume={84},

pages={108609},

year={2024},

publisher={Elsevier}

}

@inproceedings{nishimura2021ontologies,

title={Ontologies of action and object in home environment towards injury prevention},

author={Nishimura, Satoshi and Egami, Shusaku and Ugai, Takanori and Oono, Mikiko and Kitamura, Koji and Fukuda, Ken},

booktitle={The 10th International Joint Conference on Knowledge Graphs},

pages={126--130},

year={2021}

}

@article{zga2022visual,

title={Visual relationship extraction in images and a semantic interpretation with ontologies},

author={Zga, Adel and Nini, Brahim},

journal={International Journal of Intelligent Information and Database Systems},

volume={15},

number={2},

pages={223--247},

year={2022},

publisher={Inderscience Publishers (IEL)}

}

@article{chen2022graph,

title={Graph-based linguistic and visual information integration for on-site occupational hazards identification},

author={Chen, Shi and Demachi, Kazuyuki and Dong, Feiyan},

journal={Automation in Construction},

volume={137},

pages={104191},

year={2022},

publisher={Elsevier}

}

@article{li2022computer,

title={Computer Vision-Based Hazard Identification of Construction Site Using Visual Relationship Detection and Ontology},

author={Li, Yange and Wei, Han and Han, Zheng and Jiang, Nan and Wang, Weidong and Huang, Jianling},

journal={Buildings},

volume={12},

number={6},

pages={857},

year={2022},

publisher={MDPI}

}

@inproceedings{belkebir2022integrating,

title={Integrating Ontology with Imaging and Artificial Vision for a High-Level Semantic: A Review},

author={Belkebir, Malak and Maarouk, Toufik Messaoud and Nini, Brahim},

booktitle={Proceedings of the 2nd International Conference on Emerging Technologies and Intelligent Systems: ICETIS 2022, Volume 2},

pages={32--41},

year={2022},

organization={Springer}

}

@article{parveen2024fuzzy,

title={Fuzzy-Ontology Based Knowledge Driven Disease Risk Level Prediction with Optimization Assisted Ensemble Classifier},

author={Parveen, Huma and Rizvi, Syed Wajahat Abbas and Boddu, Raja Sarath Kumar},

journal={Data & Knowledge Engineering},

pages={102278},

year={2024},

publisher={Elsevier}

}

@article{zeng2024conse,

title={ConSE: An ontology for visual representation and semantic enrichment of digital images in construction sites},

author={Zeng, Cheng and Hartmann, Timo and Ma, Leyuan},

journal={Advanced Engineering Informatics},

volume={60},

pages={102446},

year={2024},

publisher={Elsevier}

}

@inproceedings{borst1997ontology,

title={An ontology approach to product disassembly},

author={Borst, Pim and Akkermans, Hans},

booktitle={Knowledge Acquisition, Modeling and Management: 10th European Workshop, EKAW'97 Sant Feliu de Guixols, Catalonia, Spain October 15--18, 1997 Proceedings 10},

pages={33--48},

year={1997},

organization={Springer}

}

@article{abusukhon2023iot,

title={IOT Bracelets for Guiding Blind People in an Indoor Environment},

author={Abusukhon, Ahmad},

journal={Journal of Communications Software and Systems},

volume={19},

number={2},

pages={114--125},

year={2023},

publisher={Udruga za komunikacijske i informacijske tehnologije, Fakultet~…}

}

@article{hadri2023friendy,

title={Friendy: A Deep Learning based Framework for Assisting in Young Autistic Children Psychotherapy Interventions},

author={Hadri, Sid Ahmed and Bouramoul, Abdelkrim},

journal={Journal of Communications Software and Systems},

volume={19},

number={1},

pages={30--38},

year={2023},

publisher={Udruga za komunikacijske i informacijske tehnologije, Fakultet~…}

}

@article{rodrigues2023fusion,

title={Fusion Object Detection and Action Recognition to Predict Violent Action},

author={Rodrigues, Nelson RP and da Costa, Nuno MC and Melo, C{'e}sar and Abbasi, Ali and Fonseca, Jaime C and Cardoso, Paulo and Borges, Jo{~a}o},

journal={Sensors},

volume={23},

number={12},

pages={5610},

year={2023},

publisher={MDPI}

}

@article{ellouze2023semantic,

title={Semantic analysis based on ontology and deep learning for a chatbot to assist persons with personality disorders on Twitter},

author={Ellouze, Mourad and Hadrich Belguith, Lamia},

journal={Behaviour & Information Technology},

pages={1--20},

year={2023},

publisher={Taylor & Francis}

}

@inproceedings{redmon2016you,

title={You only look once: Unified, real-time object detection},

author={Redmon, Joseph and Divvala, Santosh and Girshick, Ross and Farhadi, Ali},

booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},

pages={779--788},

year={2016}

}

@article{jiang2022review,

title={A Review of Yolo algorithm developments},

author={Jiang, Peiyuan and Ergu, Daji and Liu, Fangyao and Cai, Ying and Ma, Bo},

journal={Procedia Computer Science},

volume={199},

pages={1066--1073},

year={2022},

publisher={Elsevier}

}

@inproceedings{shi2021detection,

title={Detection of flying birds in airport monitoring based on improved YOLOv5},

author={Shi, Xiaohang and Hu, Jun and Lei, Xueyue and Xu, Shiyou},

booktitle={2021 6th International Conference on Intelligent Computing and Signal Processing (ICSP)},

pages={1446--1451},

year={2021},

organization={IEEE}

}

@article{guo2022msft,

title={Msft-yolo: Improved yolov5 based on transformer for detecting defects of steel surface},

author={Guo, Zexuan and Wang, Chensheng and Yang, Guang and Huang, Zeyuan and Li, Guo},

journal={Sensors},

volume={22},

number={9},

pages={3467},

year={2022},

publisher={MDPI}

}

@article{jocher2020ultralytics,

title={ultralytics/yolov5: v3. 1-bug fixes and performance improvements},

author={Jocher, Glenn and Stoken, Alex and Borovec, Jirka and Changyu, Liu and Hogan, Adam and Diaconu, Laurentiu and Ingham, Francisco and Poznanski, Jake and Fang, Jiacong and Yu, Lijun and others},

journal={Zenodo},

year={2020}

}

@article{zhou2017places,

title={Places: A 10 million image database for scene recognition},

author={Zhou, Bolei and Lapedriza, Agata and Khosla, Aditya and Oliva, Aude and Torralba, Antonio},

journal={IEEE transactions on pattern analysis and machine intelligence},

volume={40},

number={6},

pages={1452--1464},

year={2017},

publisher={IEEE}

}

@inproceedings{rezatofighi2019generalized,

title={Generalized intersection over union: A metric and a loss for bounding box regression},

author={Rezatofighi, Hamid and Tsoi, Nathan and Gwak, JunYoung and Sadeghian, Amir and Reid, Ian and Savarese, Silvio},

booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},

pages={658--666},

year={2019}

}

@article{krishna2017visual,

title={Visual genome: Connecting language and vision using crowdsourced dense image annotations},

author={Krishna, Ranjay and Zhu, Yuke and Groth, Oliver and Johnson, Justin and Hata, Kenji and Kravitz, Joshua and Chen, Stephanie and Kalantidis, Yannis and Li, Li-Jia and Shamma, David A and others},

journal={International journal of computer vision},

volume={123},

pages={32--73},

year={2017},

publisher={Springer}

}

@inproceedings{diomataris2021grounding,

title={Grounding consistency: Distilling spatial common sense for precise visual relationship detection},

author={Diomataris, Markos and Gkanatsios, Nikolaos and Pitsikalis, Vassilis and Maragos, Petros},

booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},

pages={15911--15920},

year={2021}

}

@inproceedings{tom1993toward,

title={Toward principles for the design of ontologies used for knowledge sharing},

author={Tom, Gruber},

booktitle={Int. Workshop on Formal Ontology, 1993},

year={1993}

}

@article{uschold1996ontologies,

title={Ontologies: Principles, methods and applications},

author={Uschold, Mike and Gruninger, Michael},

journal={The knowledge engineering review},

volume={11},

number={2},

pages={93--136},

year={1996},

publisher={Cambridge University Press}

}

@article{valantine2015national,

title={National Institutes of Health addresses the science of diversity},

author={Valantine, Hannah A and Collins, Francis S},

journal={Proceedings of the National Academy of Sciences},

volume={112},

number={40},

pages={12240--12242},

year={2015},

publisher={National Acad Sciences}

}

@article{genesereth1987logical,

title={Logical foundations of},

author={Genesereth, Michael R and Nilsson, Nils J},

journal={Artificial Intelligence. New York: Morgan Kaufmann Publishers},

year={1987}

}

@article{jean2021python,

title={The Python language: Adopt a snake!},

author={Jean-Baptiste, Lamy and Jean-Baptiste, Lamy},

journal={Ontologies with Python: Programming OWL 2.0 Ontologies with Python and Owlready2},

pages={9--48},

year={2021},

publisher={Springer}

}

@article{krotzsch2012description,

title={A description logic primer},

author={Kr{"o}tzsch, Markus and Simancik, Frantisek and Horrocks, Ian},

journal={arXiv preprint arXiv:1201.4089},

year={2012}

}

@inproceedings{o2008swrlapi,

title={The SWRLAPI: A Development Environment for Working with SWRL Rules.},

author={O'Connor, Martin J and Shankar, Ravi D and Musen, Mark A and Das, Amar K and Nyulas, Csongor},

booktitle={OWLED},

year={2008}

}

@inproceedings{sigurdsson2016hollywood,

title={Hollywood in homes: Crowdsourcing data collection for activity understanding},

author={Sigurdsson, Gunnar A and Varol, G{"u}l and Wang, Xiaolong and Farhadi, Ali and Laptev, Ivan and Gupta, Abhinav},

booktitle={Computer Vision--ECCV 2016: 14th European Conference, Amsterdam, The Netherlands, October 11--14, 2016, Proceedings, Part I 14},

pages={510--526},

year={2016},

organization={Springer}

}

@inproceedings{xu2015can,

title={Can humans fly? action understanding with multiple classes of actors},

author={Xu, Chenliang and Hsieh, Shao-Hang and Xiong, Caiming and Corso, Jason J},

booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},

pages={2264--2273},

year={2015}

}

@article{sirin2007pellet,

title={Pellet: A practical owl-dl reasoner},

author={Sirin, Evren and Parsia, Bijan and Grau, Bernardo Cuenca and Kalyanpur, Aditya and Katz, Yarden},

journal={Journal of Web Semantics},

volume={5},

number={2},

pages={51--53},

year={2007},

publisher={Elsevier}

}

@inproceedings{o2009sqwrl,

title={SQWRL: a query language for OWL.},

author={O'Connor, Martin J and Das, Amar K},

booktitle={OWLED},

volume={529},

pages={1--8},

year={2009}

}




DOI: https://doi.org/10.31449/inf.v48i14.6271

Creative Commons License
This work is licensed under a Creative Commons Attribution 3.0 License.