Evaluation of Shallow Convolutional Neural Network in Open-World Chart Image Classification

Filip Bajić, Marija Habijan, Krešimir Nenadić

Abstract


Data's role is pivotal in the era of internet technologies, but unstructured data poses comprehension challenges. Data visualizations like charts have emerged as crucial tools for condensing complex information. Classifying charts and applying various processing techniques are vital to interpreting visual data. Traditional chart image classification methods rely on predefined rules and have limited accuracy. The advent of support vector machines (SVMs) and convolutional neural networks (CNNs) significantly improved the accuracy of these methods. This research evaluates our previously introduced Shallow convolutional neural network (SCNN) architecture for chart image classification, comprising four convolutional layers, two max-pooling layers, and one fully-connected layer. The network achieves state-of-the-art results, requiring smaller datasets and reduced computational resources. When two networks are combined into Siamese SCNN (SSCNN), emphasizing generalization, it achieves high accuracy with small datasets and excels in open-set classification. The evaluation process encompasses the utilization of six publicly available datasets.


Full Text:

PDF

References


@article{1,

title={Reverse‐Engineering Visualizations: Recovering Visual Encodings from Chart Images},

author={Jorge Poco and Jeffrey Heer},

journal={Computer Graphics Forum},

year={2017},

volume={36},

}

@article{2,

title={Review of chart image detection and classification},

author={Filip Baji{'c} and Josip Job},

journal={International Journal on Document Analysis and Recognition (IJDAR)},

year={2023},

pages={1-22},

}

@article{3,

title={Data Visualization Classification Using Simple Convolutional Neural Network Model},

author={Filip Baji{'c} and Josip Job and Kre{vs}imir Nenadi{'c}},

journal={International Journal of Electrical and Computer Engineering},

year={2020},

volume={11},

pages={43-51},

}

@article{4,

title={Chart Classification Using Simplified VGG Model},

author={Filip Baji{'c} and Josip Job and Kre{vs}imir Nenadi{'c}},

journal={2019 International Conference on Systems, Signals and Image Processing (IWSSIP)},

year={2019},

pages={229-233},

}

@inproceedings{5,

title={FigureSeer: Parsing Result-Figures in Research Papers},

author={Noah Siegel and Zachary Horvitz and Roie Levin and Santosh Kumar Divvala and Ali Farhadi},

booktitle={European Conference on Computer Vision},

year={2016},

}

@article{6,

title={Chart decoder: Generating textual and numeric information from chart images automatically},

author={Wenjing Dai and M. Wang and Zhibin Niu and Jiawan Zhang},

journal={J. Vis. Lang. Comput.},

year={2018},

volume={48},

pages={101-109},

}

@article{7,

title={Chart-Text: A Fully Automated Chart Image Descriptor},

author={Abhijit Balaji and Thuvaarakkesh Ramanathan and Venkateshwarlu Sonathi},

journal={ArXiv},

year={2018},

volume={abs/1812.10636},

}

@article{8,

title={VizByWiki: Mining Data Visualizations from the Web to Enrich News Articles},

author={Allen Yilun Lin and Joshua Ford and Eytan Adar and Brent J. Hecht},

journal={Proceedings of the 2018 World Wide Web Conference},

year={2018},

}

@article{9,

title={Visualizing for the Non‐Visual: Enabling the Visually Impaired to Use Visualization},

author={Jinho Choi and Sanghun Jung and Deok Gun Park and Jaegul Choo and Niklas Elmqvist},

journal={Computer Graphics Forum},

year={2019},

volume={38},

}

@article{10,

title={DocFigure: A Dataset for Scientific Document Figure Classification},

author={K. V. Jobin and Ajoy Mondal and C. V. Jawahar},

journal={2019 International Conference on Document Analysis and Recognition Workshops (ICDARW)},

year={2019},

volume={1},

pages={74-79},

}

@article{11,

title={Data Extraction from Charts via Single Deep Neural Network},

author={Xiaoyi Liu and Diego Klabjan and Patrick N. Bless},

journal={ArXiv},

year={2019},

volume={abs/1906.11906},

}

@article{12,

title={ChartSense: Interactive Data Extraction from Chart Images},

author={Daekyoung Jung and Wonjae Kim and Hyunjoo Song and Jeongin Hwang and Bongshin Lee and Bo Hyoung Kim and Jinwook Seo},

journal={Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems},

year={2017},

}

@inproceedings{13,

title={Convolutional neural network based chart image classification},

author={Jihen Amara and Pawandeep Kaur and Michael Owonibi and Bassem Bouaziz},

year={2017},

}

@article{14,

title={Evaluation of Convolutional Neural Network Architectures for Chart Image Classification},

author={Paulo Chagas and Rafael Daisuke Akiyama and Aruanda Sim{~o}es Goncalves Meiguins and Carlos G. R. Santos and Filipe de Oliveira Saraiva and Bianchi Serique Meiguins and Jefferson Magalh{~a}es de Morais},

journal={2018 International Joint Conference on Neural Networks (IJCNN)},

year={2018},

pages={1-8},

}

@article{15,

title={Document Image Classification: Towards Assisting Visually Impaired},

author={K. C. Shahira and A. Lijiya},

journal={TENCON 2019 - 2019 IEEE Region 10 Conference (TENCON)},

year={2019},

pages={852-857},

}

@inproceedings{16,

title={Combining Image and Caption Analysis for Classifying Charts in Biodiversity Texts},

author={Pawandeep Kaur and Dora Kiesel},

booktitle={VISIGRAPP},

year={2020},

}

@article{17,

title={Chart classification by combining deep convolutional networks and deep belief networks},

author={Xiao Liu and Binbin Tang and Zhenyang Wang and Xianghua Xu and Shiliang Pu and Dapeng Tao and Mingli Song},

journal={2015 13th International Conference on Document Analysis and Recognition (ICDAR)},

year={2015},

pages={801-805},

}

@article{18,

title={Architecture Proposal for Data Extraction of Chart Images Using Convolutional Neural Network},

author={Paulo Roberto Silva Chagas Junior and Alexandre Abreu de Freitas and Rafael Daisuke Akiyama and Brunelli Pinto Miranda and Tiago Ara{'u}jo and Carlos G. R. Santos and Bianchi Serique Meiguins and Jefferson Magalh{~a}es de Morais},

journal={2017 21st International Conference Information Visualisation (IV)},

year={2017},

pages={318-323},

}

@inproceedings{19,

title={A Saliency-based Convolutional Neural Network for Table and Chart Detection in Digitized Documents},

author={Isaak Kavasidis and Sergio Palazzo and Concetto Spampinato and Carmelo Pino and Daniela Giordano and Daniele Giuffrida and Paolo Messina},

booktitle={International Conference on Image Analysis and Processing},

year={2018},

}

@article{20,

title={DeepGraphNet: Grafiklerin Sınıflandırılmasında Derin {"O}ğrenme Modelleri},

author={G{"o}khan Altan},

journal={European Journal of Science and Technology},

year={2019},

}

@inproceedings{21,

title={An Image Classification Tool of Wikimedia Commons},

author={Sisi Huang},

year={2020},

}

@article{22,

title={Multi-label classification of line chart images using convolutional neural networks},

author={Cem Kosemen and Derya Birant},

journal={SN Applied Sciences},

year={2020},

volume={2},

pages={1-20},

}

@inproceedings{23,

title={Chart-Type Classification Using Convolutional Neural Network for Scholarly Figures},

author={Takeo Ishihara and Kento Morita and Nobu C. Shirai and Tetsushi Wakabayashi and Wataru Ohyama},

booktitle={Asian Conference on Pattern Recognition},

year={2019},

}

@inproceedings{24,

title={A Machine Learning Approach for Semantic Structuring of Scientific Charts in Scholarly Documents},

author={Rabah A. Al-Zaidy and C. Lee Giles},

booktitle={AAAI Conference on Artificial Intelligence},

year={2017},

}

@inproceedings{25,

title={Pie Chart or Pizza: Identifying Chart Types and Their Virality on Twitter},

author={Pavlos Vougiouklis and Les Carr and Elena Paslaru Bontas Simperl},

booktitle={International Conference on Web and Social Media},

year={2020},

}

@article{26,

title={A Real-World Approach on the Problem of Chart Recognition Using Classification, Detection and Perspective Correction},

author={Tiago Ara{'u}jo and Paulo Chagas and Jo{~a}o Bernardo Alves and Carlos G. R. Santos and Beatriz Sousa Santos and Bianchi Serique Meiguins},

journal={Sensors (Basel, Switzerland)},

year={2020},

volume={20},

}

@inproceedings{27,

title={BarChartAnalyzer: Digitizing Images of Bar Charts},

author={Komal Dadhich and Siri Chandana Daggubati and Jaya Sreevalsan-Nair},

booktitle={International Conference on Image Processing and Vision Engineering},

year={2021},

}

@article{28,

title={Towards an efficient framework for Data Extraction from Chart Images},

author={Weihong Ma and Hesuo Zhang and Shuang Yan and Guang Yao and Yichao Huang and Hui Li and Yaqiang Wu and Lianwen Jin},

journal={ArXiv},

year={2021},

volume={abs/2105.02039},

}

@article{29,

title={ChartReader: Automatic Parsing of Bar-Plots},

author={Chinmayee Rane and Seshasayee Mahadevan Subramanya and Devi Sandeep Endluri and Jian Wu and C. Lee Giles},

journal={2021 IEEE 22nd International Conference on Information Reuse and Integration for Data Science (IRI)},

year={2021},

pages={318-325},

}

@article{30,

title={Image & Graphic Reader},

author={Ingo Redeke},

journal={Proceedings 2001 International Conference on Image Processing (Cat. No.01CH37205)},

year={2001},

volume={1},

pages={806-809 vol.1},

}

@article{31,

title={ReVision: automated classification, analysis and redesign of chart images},

author={Manolis Savva and Nicholas Kong and Arti Chhajta and Li Fei-Fei and Maneesh Agrawala and Jeffrey Heer},

journal={Proceedings of the 24th annual ACM symposium on User interface software and technology},

year={2011},

}

@article{32,

title={View: Visual Information Extraction Widget for improving chart images accessibility},

author={Jinglun Gao and Yin Zhou and Kenneth E. Barner},

journal={2012 19th IEEE International Conference on Image Processing},

year={2012},

pages={2865-2868},

}

@article{33,

title={Model-Based Recognition and Extraction of Information from Chart Images},

author={Ales Mishchenko and Natalia Vassilieva},

journal={J. Multim. Process. Technol.},

year={2011},

volume={2},

pages={76-89},

}

@inproceedings{34,

title={Model-Based Chart Image Classification},

author={Ales Mishchenko and Natalia Vassilieva},

booktitle={International Symposium on Visual Computing},

year={2011},

}

@article{35,

title={Automated analysis of line plots in documents},

author={Rathin Radhakrishnan Nair and Nishant Sankaran and Ifeoma Nwogu and Venu Govindaraju},

journal={2015 13th International Conference on Document Analysis and Recognition (ICDAR)},

year={2015},

pages={796-800},

}

@article{36,

title={Statistical graph classification in intelligent mathematics problem solving system for high school student},

author={Yafei Shi and Yantao Wei and Ting Wu and Qingtang Liu},

journal={2017 12th International Conference on Computer Science and Education (ICCSE)},

year={2017},

pages={645-650},

}

@article{37,

title={Challenges in chart image classification: a comparative study of different deep learning methods},

author={Jennil Thiyam and Sanasam Ranbir Singh and Prabin Kumar Bora},

journal={Proceedings of the 21st ACM Symposium on Document Engineering},

year={2021},

}

@inproceedings{38,

title={A Survey and Approach to Chart Classification},

author={Anurag Dhote and M. Hammad Javed and David S. Doermann},

booktitle={ICDAR Workshops},

year={2023},

}

@article{39,

title={Machine Learning Classification Algorithms to Recognize Chart Types in Portable Document Format (PDF) Files},

author={V. Karthikeyani and S. Nagarajan},

journal={International Journal of Computer Applications},

year={2012},

volume={39},

pages={1-5},

}

@article{40,

title={Graphical Figure Classification Using Data Fusion for Integrating Text and Image Features},

author={Beibei Cheng and Ronald Joe Stanley and Sameer Kiran Antani and George R. Thoma},

journal={2013 12th International Conference on Document Analysis and Recognition},

year={2013},

pages={693-697},

}

@article{41,

title={Beagle: Automated Extraction and Interpretation of Visualizations from the Web},

author={Leilani Battle and Peitong Duan and Zachery Miranda and Dana Mukusheva and Remco Chang and Michael Stonebraker},

journal={Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems},

year={2017},

}

@article{42,

title={ICDAR 2019 Competition on Harvesting Raw Tables from Infographics (CHART-Infographics)},

author={Kenny Davila and Bhargava Urala Kota and Srirangaraj Setlur and Venu Govindaraju and Chris Tensmeyer and Sumit Shekhar and Ritwick Chaudhry},

journal={2019 International Conference on Document Analysis and Recognition (ICDAR)},

year={2019},

pages={1594-1599},

}

@inproceedings{43,

title={AutoChart: A Dataset for Chart-to-Text Generation Task},

author={Jiawen Zhu and Jinye Ran and Roy Ka-Wei Lee and Kenny Choo and Zhi Li},

booktitle={Recent Advances in Natural Language Processing},

year={2021},

}

@article{44,

title={ICPR 2022: Challenge on Harvesting Raw Tables from Infographics (CHART-Infographics)},

author={Kenny Davila and Fei Xu and Saleem Ahmed and David A. Mendoza and Srirangaraj Setlur and Venu Govindaraju},

journal={2022 26th International Conference on Pattern Recognition (ICPR)},

year={2022},

pages={4995-5001},

}

@article{45,

title={Chart Classification Using Siamese CNN},

author={Filip Baji{'c} and Josip Job},

journal={Journal of Imaging},

year={2021},

volume={7},

}

@article{46,

title={Effect of attention and triplet loss on chart classification: a study on noisy charts and confusing chart pairs},

author={Jennil Thiyam and Sanasam Ranbir Singh and Prabin Kumar Bora},

journal={Journal of Intelligent Information Systems},

year={2022},

volume={60},

pages={731 - 758},

}

@article{47,

title={A Multi-Purpose Shallow Convolutional Neural Network for Chart Images},

author={Filip Baji{'c} and Ognjen Orel and Marija Habijan},

journal={Sensors (Basel, Switzerland)},

year={2022},

volume={22},

}

@article{48,

title={ChartDETR: A Multi-shape Detection Network for Visual Chart Recognition},

author={Wenyuan Xue and Dapeng Chen and Baosheng Yu and Yifei Chen and Sai Zhou and Wei Peng},

journal={ArXiv},

year={2023},

volume={abs/2308.07743},

}

@misc{49,

title={ChartDS - chartDataset.zip. figshare},

author={F. Bajić},

journal={ArXiv},

year={2022}

}

@misc{50,

title={ChartDS 2022 - chartDataset.zip Srce Dabar},

author={Filip Baji{'c}},

year={2022}

}

@article{51,

title={Linnaeus 5 Dataset for Machine Learning},

author={G. Chaladze and Kalatozishvili L},

year={2017}

}

@inproceedings{52,

title={Learning Multiple Layers of Features from Tiny Images},

author={Alex Krizhevsky},

year={2009},

}

@misc{53,

title={Handbook of Face Recognition},

author={Stan Z. Li and Anil K. Jain},

year={2011},

}

@misc{54,

title={ChartDataset2023: Introducing a Synthetic Dataset Featuring Various Chart Types for Chart Identification and Visualization},

author={F. Bajić, M. Habijan and K. Nenadić},

year={2023},

}

@article{55,

title={Open Set Source Camera Attribution},

author={Filipe de Oliveira Costa and Michael Eckmann and Walter J. Scheirer and Anderson Rocha},

journal={2012 25th SIBGRAPI Conference on Graphics, Patterns and Images},

year={2012},

pages={71-78},

}

@ARTICLE{56,

author={Scheirer, Walter J. and de Rezende Rocha, Anderson and Sapkota, Archana and Boult, Terrance E.},

journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},

title={Toward Open Set Recognition},

year={2013},

volume={35},

number={7},

pages={1757-1772},

}

@article{57,

title={The Extreme Value Machine},

author={Ethan M. Rudd and Lalit P. Jain and Walter J. Scheirer and Terrance E. Boult},

journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},

year={2015},

volume={40},

pages={762-768},

}

@inproceedings{58,

title={Learning and the Unknown: Surveying Steps toward Open World Recognition},

author={Terrance E. Boult and Steve Cruz and Akshay Raj Dhamija and Manuel G{"u}nther and James Henrydoss and Walter J. Scheirer},

booktitle={AAAI Conference on Artificial Intelligence},

year={2019},

}

@article{59,

title={Sparse Representation-Based Open Set Recognition},

author={He Zhang and Vishal M. Patel},

journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},

year={2017},

volume={39},

pages={1690-1696},

}

@article{60,

title={Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks},

author={Shaoqing Ren and Kaiming He and Ross B. Girshick and Jian Sun},

journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},

year={2015},

volume={39},

pages={1137-1149},

}

@article{61,

title={Multi-Level Semantic Feature Augmentation for One-Shot Learning},

author={Z. Chen and Yanwei Fu and Yinda Zhang and Yu-Gang Jiang and X. Xue and Leonid Sigal},

journal={IEEE Transactions on Image Processing},

year={2018},

volume={28},

pages={4594-4605},

}

@article{62,

title={Recent Advances in Zero-Shot Recognition: Toward Data-Efficient Understanding of Visual Content},

author={Yanwei Fu and Tao Xiang and Yu-Gang Jiang and X. Xue and Leonid Sigal and Shaogang Gong},

journal={IEEE Signal Processing Magazine},

year={2017},

volume={35},

pages={112-125},

}




DOI: https://doi.org/10.31449/inf.v48i6.5660

Creative Commons License
This work is licensed under a Creative Commons Attribution 3.0 License.