@inproceedings{Jenadeleh2023-06Relax-66792,
  year={2023},
  title={Relaxed forced choice improves performance of visual quality assessment methods},
  author={Jenadeleh, Mohsen and Zagermann, Johannes and Reiterer, Harald and Reips, Ulf-Dietrich and Hamzaoui, Raouf and Saupe, Dietmar},
  note={Funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) -- Project ID 496858717 and DFG Project ID 251654672 -- TRR 161 (Projects A05 and C01). This research was also kindly supported by the Zukunftskolleg, the University of Konstanz, with funding from the Excellence Strategy of the German Federal and State Governments.}
}

<rdf:RDF
    xmlns:dcterms="http://purl.org/dc/terms/"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
    xmlns:bibo="http://purl.org/ontology/bibo/"
    xmlns:dspace="http://digital-repositories.org/ontologies/dspace/0.1.0#"
    xmlns:foaf="http://xmlns.com/foaf/0.1/"
    xmlns:void="http://rdfs.org/ns/void#"
    xmlns:xsd="http://www.w3.org/2001/XMLSchema#" > 
  <rdf:Description rdf:about="https://kops.uni-konstanz.de/server/rdf/resource/123456789/66792">
    <dc:contributor>Hamzaoui, Raouf</dc:contributor>
    <dcterms:isPartOf rdf:resource="https://kops.uni-konstanz.de/server/rdf/resource/123456789/52"/>
    <dc:contributor>Reiterer, Harald</dc:contributor>
    <dc:contributor>Reips, Ulf-Dietrich</dc:contributor>
    <dcterms:abstract>In image quality assessment, a collective visual quality score for an image or video is obtained from the individual ratings of many subjects. One commonly used format for these experiments is the two-alternative forced choice method. Two stimuli with the same content but differing visual quality are presented sequentially or side-by-side. Subjects are asked to select the one of better quality, and when uncertain, they are required to guess. The relaxed alternative forced choice format aims to reduce the cognitive load and the noise in the responses due to the guessing by providing a third response option, namely, ``not sure''. This work presents a large and comprehensive crowdsourcing experiment to compare these two response formats: the one with the ``not sure'' option and the one without it. To provide unambiguous ground truth for quality evaluation, subjects were shown pairs of images with differing numbers of dots and asked each time to choose the one with more dots. Our crowdsourcing study involved 254 participants and was conducted using a within-subject design. Each participant was asked to respond to 40 pair comparisons with and without the ``not sure'' response option and completed a questionnaire to evaluate their cognitive load for each testing condition. The experimental results show that the inclusion of the ``not sure'' response option in the forced choice method reduced mental load and led to models with better data fit and correspondence to ground truth. We also tested for the equivalence of the models and found that they were different. The dataset is available at http://database.mmsp-kn.de/cogvqa-database.html.</dcterms:abstract>
    <dspace:hasBitstream rdf:resource="https://kops.uni-konstanz.de/bitstream/123456789/66792/1/Jenadeleh_2-13oqtgwzu92fb8.pdf"/>
    <bibo:uri rdf:resource="https://kops.uni-konstanz.de/handle/123456789/66792"/>
    <dcterms:rights rdf:resource="https://rightsstatements.org/page/InC/1.0/"/>
    <void:sparqlEndpoint rdf:resource="http://localhost/fuseki/dspace/sparql"/>
    <dc:contributor>Zagermann, Johannes</dc:contributor>
    <dcterms:title>Relaxed forced choice improves performance of visual quality assessment methods</dcterms:title>
    <dc:creator>Reips, Ulf-Dietrich</dc:creator>
    <dc:rights>terms-of-use</dc:rights>
    <dc:creator>Zagermann, Johannes</dc:creator>
    <dspace:isPartOfCollection rdf:resource="https://kops.uni-konstanz.de/server/rdf/resource/123456789/28"/>
    <dc:contributor>Saupe, Dietmar</dc:contributor>
    <dc:language>eng</dc:language>
    <dc:creator>Hamzaoui, Raouf</dc:creator>
    <dcterms:isPartOf rdf:resource="https://kops.uni-konstanz.de/server/rdf/resource/123456789/28"/>
    <dc:creator>Reiterer, Harald</dc:creator>
    <dc:contributor>Jenadeleh, Mohsen</dc:contributor>
    <dcterms:available rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2023-05-04T09:50:02Z</dcterms:available>
    <foaf:homepage rdf:resource="http://localhost:8080/"/>
    <dc:date rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2023-05-04T09:50:02Z</dc:date>
    <dc:creator>Jenadeleh, Mohsen</dc:creator>
    <dcterms:hasPart rdf:resource="https://kops.uni-konstanz.de/bitstream/123456789/66792/1/Jenadeleh_2-13oqtgwzu92fb8.pdf"/>
    <dc:creator>Saupe, Dietmar</dc:creator>
    <dcterms:issued>2023-06</dcterms:issued>
    <dspace:isPartOfCollection rdf:resource="https://kops.uni-konstanz.de/server/rdf/resource/123456789/52"/>
  </rdf:Description>
</rdf:RDF>