@inproceedings{MettesICCV17,
title = {Spatial-Aware Object Embeddings for Zero-Shot Localization and Classification of Actions},
author = {Pascal Mettes and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mettes-spatial-aware-iccv2017.pdf
https://github.com/psmmettes/spatial-aware-object-embeddings},
year = {2017},
date = {2017-10-01},
booktitle = {ICCV},
address = {Venice, Italy},
abstract = {We aim for zero-shot localization and classification of human actions in video. Where traditional approaches rely on global attribute or object classification scores for their zero-shot knowledge transfer, our main contribution is a spatial-aware object embedding. To arrive at spatial awareness, we build our embedding on top of freely available actor and object detectors. Relevance of objects is determined in a word embedding space and further enforced with estimated spatial preferences. Besides local object awareness, we also embed global object awareness into our embedding to maximize actor and object interaction. Finally, we exploit the object positions and sizes in the spatial-aware embedding to demonstrate a new spatio-temporal action retrieval scenario with composite queries. Action localization and classification experiments on four contemporary action video datasets support our proposal. Apart from state-of-the-art results in the zero-shot localization and classification settings, our spatial-aware embedding is even competitive with recent supervised action localization alternatives.},
note = {Oral presentation, top 2.1%},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{CappalloMM17,
title = {Future-Supervised Retrieval of Unseen Queries for Live Video},
author = {Spencer Cappallo and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/cappallo-future-supervised-mm2017.pdf},
year = {2017},
date = {2017-10-01},
booktitle = {MM},
address = {Mountain View, USA},
abstract = {Live streaming video presents new challenges for retrieval and content understanding. Its live nature means that video representations should be relevant to current content, and not necessarily to past content. We investigate retrieval of previously unseen queries for live video content. Drawing from existing whole-video techniques, we focus on adapting image-trained semantic models to the video domain. We introduce the use of future frame representations as a supervision signal for learning temporally aware semantic representations on unlabeled video data. Additionally, we introduce an approach for broadening a query?s representation within a pre-constructed semantic space, with the aim of increasing overlap between embedded visual semantics and the query semantics. We demonstrate the efficacy of these contributions for unseen query retrieval on live videos. We further explore their applicability to tasks such as no example, whole-video action classification and no-example live video action prediction, and demonstrate state of the art results.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{MettesBMVC17,
title = {Localizing Actions from Video Labels and Pseudo-Annotations},
author = {Pascal Mettes and Cees G M Snoek and Shih-Fu Chang},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mettes-pseudo-annotations-bmvc2017.pdf},
year = {2017},
date = {2017-09-01},
booktitle = {BMVC},
address = {London, UK},
abstract = {The goal of this paper is to determine the spatio-temporal location of actions in video. Where training from hard to obtain box annotations is the norm, we propose an intuitive and effective algorithm that localizes actions from their class label only. We are inspired by recent work showing that unsupervised action proposals selected with human point-supervision perform as well as using expensive box annotations. Rather than asking users to provide point supervision, we propose fully automatic visual cues that replace manual point annotations. We call the cues pseudo-annotations, introduce five of them, and propose a correlation metric for automatically selecting and combining them. Thorough evaluation on challenging action localization datasets shows that we reach results comparable to results with full box supervision. We also show that pseudo-annotations can be leveraged during testing to improve weakly- and strongly-supervised localizers.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{LiCVPR17,
title = {Tracking by Natural Language Specification},
author = {Zhenyang Li and Ran Tao and Efstratios Gavves and Cees G M Snoek and Arnold W M Smeulders},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/li-tracking-language-cvpr2017.pdf},
year = {2017},
date = {2017-07-01},
booktitle = {CVPR},
address = {Honolulu, USA},
abstract = {This paper strives to track a target object in a video. Rather than specifying the target in the first frame of a video by a bounding box, we propose to track the object based on a natural language specification of the target, which provides a more natural human-machine interaction as well as a means to improve tracking results. We define three variants of tracking by language specification: one relying on lingual target specification only, one relying on visual target specification based on language, and one leveraging their joint capacity. To show the potential of tracking by natural language specification we extend two popular tracking datasets with lingual descriptions and report experiments. Finally, we also sketch new tracking scenarios in surveillance and other live video streams that become feasible with a lingual specification of the target.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{MensinkICMR17,
title = {Music-Guided Video Summarization using Quadratic Assignments},
author = {Thomas Mensink and Thomas Jongstra and Pascal Mettes and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mensink-music-video-summarization-icmr2017.pdf
http://isis-data.science.uva.nl/cgmsnoek/pub/mensink-music-video-summarization-icmr2017.mp4},
year = {2017},
date = {2017-06-01},
booktitle = {ICMR},
address = {Bucharest, Romania},
abstract = {This paper aims to automatically generate a summary of an unedited video, guided by an externally provided music-track. The tempo, energy and beats in the music determine the choices and cuts in the video summarization. To solve this challenging task, we model video summarization as a quadratic assignment problem. We assign frames to the summary, using rewards based on frame interestingness, plot coherency, audio-visual match, and cut properties. Experimentally we validate our approach on the SumMe dataset. The results show that our music guided summaries are more appealing, and even outperform the current state-of-the-art summarization methods when evaluated on the F1 measure of precision and recall.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{KovvuriICPR16,
title = {Segment-based Models for Event Detection and Recounting},
author = {Rama Kovvuri and Ram Nevatia and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/kovvuri-segment-models-icpr2016.pdf},
year = {2016},
date = {2016-12-01},
booktitle = {ICPR},
address = {Cancun, Mexico},
abstract = {We present a novel approach towards web video classification and recounting that uses video segments to model an event. This approach overcomes the limitations faced by the classical video-level models such as modeling semantics, identifying informative segments in a video and background segment suppression. We posit that segment-based models are able to identify both the frequently-occurring and rarer patterns in an event effectively, despite being trained on only a fraction of the training data. Our framework employs a discriminative approach to optimize our models in distributed and data-driven fashion while maintaining semantic interpretability. We evaluate the effectiveness of our approach on the challenging TRECVID MEDTest 2014 dataset. We demonstrate improvements in recounting and classification, particularly in events characterized by inherent intra-class variations.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{SnoekTRECVID16,
title = {University of Amsterdam and Renmin University at TRECVID 2016: Searching Video, Detecting Events and Describing Video},
author = {Cees G M Snoek and Jianfeng Dong and Xirong Li and Xiaoxu Wang and Qijie Wei and Weiyu Lan and Efstratios Gavves and Noureldien Hussein and Dennis C Koelma and Arnold W M Smeulders},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mediamill-TRECVID2016-final.pdf},
year = {2016},
date = {2016-11-01},
booktitle = {TRECVID},
address = {Gaithersburg, USA},
abstract = {In this paper we summarize our TRECVID 2016 video recognition experiments. We participated in three tasks: video search, event detection and video description. Here we describe the tasks on event detection and video description. For event detection we explore semantic representations based on VideoStory and an ImageNet Shuffle for both zero-shot and few-example regimes. For the showcase task on video description we experiment with a deep network that predicts a visual representation from a natural language description, and use this space for the sentence matching. For generative description we enhance a neural image captioning model with Early Embedding and Late Reranking. The 2016 edition of the TRECVID benchmark has been a fruitful participation for our joint-team, resulting in the best overall result for zero- and few-example event detection as well as video description by matching and in generative mode.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{DongMM16,
title = {Early Embedding and Late Reranking for Video Captioning},
author = {Jianfeng Dong and Xirong Li and Weiyu Lan and Yujia Huo and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/dong-captioning-mm2016.pdf},
year = {2016},
date = {2016-10-01},
booktitle = {MM},
address = {Amsterdam, The Netherlands},
abstract = {This paper describes our solution for the MSR Video to Language Challenge. We start from the popular ConvNet + LSTM model, which we extend with two novel modules. One is early embedding, which enriches the current low-level input to LSTM by tag embeddings. The other is late reranking, for re-scoring generated sentences in terms of their relevance to a specific video. The modules are inspired by recent works on image captioning, repurposed and redesigned for video. As experiments on the MSR-VTT validation set show, the joint use of these two modules add a clear improvement over a non-trivial ConvNet + LSTM baseline under four performance metrics. The viability of the proposed solution is further confirmed by the blind test by the organizers. Our system is ranked at the 4th place in terms of overall performance, while scoring the best CIDEr-D, which measures the human-likeness of generated captions.},
note = {Multimedia Grand Challenge winner},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{GeestECCV16,
title = {Online Action Detection},
author = {Roeland De Geest and Efstratios Gavves and Amir Ghodrati and Zhenyang Li and Cees G M Snoek and Tinne Tuytelaars},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/geest-online-action-eccv2016.pdf
https://homes.esat.kuleuven.be/psi-archive/rdegeest/TVSeries.html},
year = {2016},
date = {2016-10-01},
booktitle = {ECCV},
address = {Amsterdam, The Netherlands},
abstract = {In online action detection, the goal is to detect the start of an action in a video stream as soon as it happens. For instance, if a child is chasing a ball, an autonomous car should recognize what is going on and respond immediately. This is a very challenging problem for four reasons. First, only partial actions are observed. Second, there is a large variability in negative data. Third, the start of the action is unknown, so it is unclear over what time window the information should be integrated. Finally, in real world data, large within-class variability exists. This problem has been addressed before, but only to some extent. Our contributions to online action detection are threefold. First, we introduce a realistic dataset composed of 27 episodes from 6 popular TV series. The dataset spans over 16 hours of footage annotated with 30 action classes, totaling 6,231 action instances. Second, we analyze and compare various baseline methods, showing this is a challenging problem for which none of the methods provides a good solution. Third, we analyze the change in performance when there is a variation in viewpoint, occlusion, truncation, etc. We introduce an evaluation protocol for fair comparison. The dataset, the baselines and the models will all be made publicly available to encourage (much needed) further research on online action detection on realistic data.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{MettesECCV16,
title = {Spot On: Action Localization from Pointly-Supervised Proposals},
author = {Pascal Mettes and Jan C van Gemert and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mettes-pointly-eccv2016.pdf
http://isis-data.science.uva.nl/mettes/hollywood2tubes.tar.gz},
year = {2016},
date = {2016-10-01},
booktitle = {ECCV},
address = {Amsterdam, The Netherlands},
abstract = {We strive for spatio-temporal localization of actions in videos. The state-of-the-art relies on action proposals at test time and selects the best one with a classifier demanding carefully annotated box annotations at train time. Annotating action boxes in video is cumbersome, tedious, and error prone. Rather than annotating boxes, we propose to annotate actions in video with points on a sparse subset of frames only. We introduce an overlap measure between action proposals and points and incorporate them all into the objective of a non-convex Multiple Instance Learning optimization. Experimental evaluation on the UCF Sports and UCF 101 datasets shows that (i) spatio-temporal proposals can be used to train classifiers while retaining the localization performance, (ii) point annotations yield results comparable to box annotations while being significantly faster to annotate, (iii) with a minimum amount of supervision our approach is competitive to the state-of-the-art. Finally, we introduce spatio-temporal action annotations on the train and test videos of Hollywood2, resulting in Hollywood2Tubes, available at tinyurl.com/hollywood2tubes.},
note = {Oral presentation, top 1.8%},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{CappalloBMVC16,
title = {Video Stream Retrieval of Unseen Queries using Semantic Memory},
author = {Spencer Cappallo and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/cappallo-videostream-bmvc2016.pdf},
year = {2016},
date = {2016-09-01},
booktitle = {BMVC},
address = {York, UK},
abstract = {Retrieval of live, user-broadcast video streams is an under-addressed and increasingly relevant challenge. The on-line nature of the problem requires temporal evaluation and the unforeseeable scope of potential queries motivates an approach which can accommodate arbitrary search queries. To account for the breadth of possible queries, we adopt a no-example approach to query retrieval, which uses a query's semantic relatedness to pre-trained concept classifiers. To adapt to shifting video content, we propose memory pooling and memory welling methods that favor recent information over long past content. We identify two stream retrieval tasks, instantaneous retrieval at any particular time and continuous retrieval over a prolonged duration, and propose means for evaluating them. Three large scale video datasets are adapted to the challenge of stream retrieval. We report results for our search methods on the new stream retrieval tasks, as well as demonstrate their efficacy in a traditional, non-streaming video task.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{KordumovaICMR16,
title = {Pooling Objects for Recognizing Scenes without Examples},
author = {Svetlana Kordumova and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/kordumova-pooling-objects-icmr2016.pdf},
year = {2016},
date = {2016-06-01},
booktitle = {ICMR},
address = {New York, USA},
abstract = {In this paper we aim to recognize scenes in images without using any scene images as training data. Different from attribute based approaches, we do not carefully select the training classes to match the unseen scene classes. Instead, we propose a pooling over ten thousand of off-the-shelf object classifiers. To steer the knowledge transfer between objects and scenes we learn a semantic embedding with the aid of a large social multimedia corpus. Our key contributions are: we are the first to investigate pooling over ten thousand object classifiers to recognize scenes without examples; we explore the ontological hierarchy of objects and analyze the influence of object classifiers from different hierarchy levels; we exploit object positions in scene images and we demonstrate a new scene retrieval scenario with complex queries. Finally, we outperform attribute representations on two challenging scene datasets, SUNAttributes and Places2.},
note = {Best paper award},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{MettesICMR16,
title = {The ImageNet Shuffle: Reorganized Pre-training for Video Event Detection},
author = {Pascal Mettes and Dennis Koelma and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mettest-imagenetshuffle-icmr2016.pdf
https://staff.fnwi.uva.nl/p.s.m.mettes/codedata.html},
year = {2016},
date = {2016-06-01},
booktitle = {ICMR},
address = {New York, USA},
abstract = {This paper strives for video event detection using a representation learned from deep convolutional neural networks. Different from the leading approaches, who all learn from the 1,000 classes defined in the ImageNet Large Scale Visual Recognition Challenge, we investigate how to leverage the complete ImageNet hierarchy for pre-training deep networks. To deal with the problems of over-specific classes and classes with few images, we introduce a bottom-up and top-down approach for reorganization of the ImageNet hierarchy based on all its 21,814 classes and more than 14 million images. Experiments on the TRECVID Multimedia Event Detection 2013 and 2015 datasets show that video representations derived from the layers of a deep neural network pre-trained with our reorganized hierarchy i) improves over standard pre-training, ii) is complementary among different reorganizations, iii) maintains the benefits of fusion with other modalities, and iv) leads to state-of-the-art event detection results. The reorganized hierarchies and their derived Caffe models are publicly available at http://tinyurl.com/imagenetshuffle.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{AgharwalWACV16,
title = {Tag-based Video Retrieval by Embedding Semantic Content in a Continuous Word Space},
author = {Arnav Agharwal and Rama Kovvuri and Ram Nevatia and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/agharwal-continuous-wacv2016.pdf},
year = {2016},
date = {2016-03-01},
booktitle = {WACV},
pages = {1--8},
address = {Lake Placid, USA},
abstract = {Content-based event retrieval in unconstrained web videos, based on query tags, is a hard problem due to large intra-class variances, and limited vocabulary and accuracy of the video concept detectors, creating a "semantic query gap". We present a technique to overcome this gap by using continuous word space representations to explicitly compute query and detector concept similarity. This not only allows for fast query-video similarity computation with implicit query expansion, but leads to a compact video representation, which allows implementation of a real-time retrieval system that can fit several thousand videos in a few hundred megabytes of memory. We evaluate the effectiveness of our representation on the challenging NIST MEDTest 2014 dataset.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{KordumovaMMM16,
title = {Exploring the Long Tail of Social Media Tags},
author = {Svetlana Kordumova and Jan C van Gemert and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/kordumova-longtail-mmm2016.pdf},
year = {2016},
date = {2016-01-01},
booktitle = {International Conference on Multimedia Modelling},
address = {Miami, USA},
abstract = {There are millions of users who tag multimedia content, generating a large vocabulary of tags. Some tags are frequent, while other tags are rarely used following a long tail distribution. For frequent tags, most of the multimedia methods that aim to automatically understand audio-visual content, give excellent results. It is not clear, however, how these methods will perform on rare tags. In this paper we investigate what social tags constitute the long tail and how they perform on two multimedia retrieval scenarios, tag relevance and detector learning. We show common valuable tags within the long tail, and by augmenting them with semantic knowledge, the performance of tag relevance and detector learning improves substantially.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{GavvesICCV15,
title = {Active Transfer Learning with Zero-Shot Priors: Reusing Past Datasets for Future Tasks},
author = {Efstratios Gavves and Thomas Mensink and Tatiana Tommasi and Cees G M Snoek and Tinne Tuytelaars},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/gavves-zero-shot-priors-iccv2015.pdf},
year = {2015},
date = {2015-12-01},
booktitle = {ICCV},
address = {Santiago, Chile},
abstract = {How can we reuse existing knowledge, in the form of available datasets, when solving a new and apparently unrelated target task from a set of unlabeled data? In this work we make a first contribution to answer this question in the context of image classification. We frame this quest as an active learning problem and use zero-shot classifiers to guide the learning process by linking the new task to the existing classifiers. By revisiting the dual formulation of adaptive SVM, we reveal two basic conditions to choose greedily only the most relevant samples to be annotated. On this basis we propose an effective active learning algorithm which learns the best possible target classification model with minimum human labeling effort. Extensive experiments on two challenging datasets show the value of our approach compared to the state-of-the-art active learning methodologies, as well as its potential to reuse past datasets with minimal effort for future tasks.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{JainICCV15,
title = {Objects2action: Classifying and localizing actions without any video example},
author = {Mihir Jain and Jan C van Gemert and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/jain-objects2action-iccv2015.pdf},
year = {2015},
date = {2015-12-01},
booktitle = {ICCV},
address = {Santiago, Chile},
abstract = {The goal of this paper is to recognize actions in video without the need for examples. Different from traditional zero-shot approaches we do not demand the design and specification of attribute classifiers and class-to-attribute mappings to allow for transfer from seen classes to unseen classes. Our key contribution is objects2action, a semantic word embedding that is spanned by a skip-gram model of thousands of object categories. Action labels are assigned to an object encoding of unseen video based on a convex combination of action and object affinities. Our semantic embedding has three main characteristics to accommodate for the specifics of actions. First, we propose a mechanism to exploit multiple-word descriptions of actions and objects. Second, we incorporate the automated selection of the most responsive objects per action. And finally, we demonstrate how to extend our zero-shot approach to the spatio-temporal localization of actions in video. Experiments on four action datasets demonstrate the potential of our approach.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{SnoekTRECVID15,
title = {Qualcomm Research and University of Amsterdam at TRECVID 2015: Recognizing Concepts, Objects, and Events in Video},
author = {Cees G M Snoek and Spencer Cappallo and Daniel Fontijne and David Julian and Dennis C Koelma and Pascal Mettes and Koen E A van de Sande and Anthony Sarah and Harro Stokman and R Blythe Towal},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mediamill-TRECVID2015-final.pdf},
year = {2015},
date = {2015-11-01},
booktitle = {TRECVID},
address = {Gaithersburg, USA},
abstract = {In this paper we summarize our TRECVID 2015 video recognition experiments. We participated in three tasks: concept detection, object localization, and event recognition, where Qualcomm Research focused on concept detection and object localization and the University of Amsterdam focused on event detection. For concept detection we start from the very deep networks that excelled in the ImageNet 2014 competition and redesign them for the purpose of video recognition, emphasizing on training data augmentation %, permutation, and dropout as well as video fine-tuning. Our entry in the localization task is based on classifying a limited number of boxes in each frame using deep learning features. The boxes are proposed by an improved version of selective search. At the core of our multimedia event detection system is an Inception-style deep convolutional neural network that is trained on the full ImageNet hierarchy with 22k categories. We propose several operations that combine and generalize the ImageNet categories to form a desirable set of (super-)categories, while still being able to train a reliable model. The 2015 edition of the TRECVID benchmark has been a fruitful participation for our team, resulting in the best overall result for concept detection, object localization and event detection.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{CappalloMM15,
title = {Image2Emoji: Zero-shot Emoji Prediction for Visual Media},
author = {Spencer Cappallo and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/cappallo-image2emoji-mm2015.pdf},
year = {2015},
date = {2015-10-01},
booktitle = {MM},
address = {Brisbane, Australia},
abstract = {We present Image2Emoji, a multi-modal approach for generating emoji labels for an image in a zero-shot manner. Different from existing zero-shot image-to-text approaches, we exploit both image and textual media to learn a semantic embedding for the new task of emoji prediction. We propose that the widespread adoption of emoji suggests a semantic universality which is well-suited for interaction with visual media. We quantify the efficacy of our proposed model on the MSCOCO dataset, and demonstrate the value of visual, textual and multi-modal prediction of emoji. We conclude the paper with three examples of the application potential of emoji in the context of multimedia retrieval.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{CappalloACM15,
title = {Query-by-Emoji Video Search},
author = {Spencer Cappallo and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/cappallo-emoji2video-mm2015.pdf},
year = {2015},
date = {2015-10-01},
booktitle = {MM},
address = {Brisbane, Australia},
abstract = {This technical demo presents Emoji2Video, a query-by-emoji interface for exploring video collections. Ideogram-based video search and representation presents an opportunity for an intuitive, visual interface and concise non-textual summary of video contents, in a form factor that is ideal for small screens. The demo allows users to build search strings comprised of ideograms which are used to query a large dataset of YouTube videos. The system returns a list of the top-ranking videos for the user query along with an emoji summary of the video contents so that users maymake an informed decision whether to view a video or refine their search terms. The ranking of the videos is done in a zero-shot, multi-modal manner that employs an embedding space to exploit semantic relationships between user-selected ideograms and the video's visual and textual content.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{GemertBMVC15,
title = {APT: Action localization proposals from dense trajectories},
author = {Jan van Gemert and Mihir Jain and Ella Gati and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/gemert-apt-proposals-bmvc2015-corrected.pdf
https://github.com/jvgemert/apt},
year = {2015},
date = {2015-09-01},
booktitle = {BMVC},
address = {Swansea, UK},
abstract = {This paper is on action localization in video with the aid of spatio-temporal proposals. To alleviate the computational expensive segmentation step of existing proposals, we propose bypassing the segmentations completely by generating proposals directly from the dense trajectories used to represent videos during classification. Our Action localization Proposals from dense Trajectories (APT) use an efficient proposal generation algorithm to handle the high number of trajectories in a video. Our spatio-temporal proposals are faster than current methods and outperform the localization and classification accuracy of current proposals on the UCF Sports, UCF 101, and MSR-II video datasets. Corrected version: we fixed a mistake in our UCF-101 ground truth. Numbers are different; conclusions are unchanged.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{NagelBMVC15,
title = {Event Fisher Vectors: Robust Encoding Visual Diversity of Visual Streams},
author = {Markus Nagel and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/nagel-event-fisher-bmvc2015.pdf},
year = {2015},
date = {2015-09-01},
booktitle = {BMVC},
address = {Swansea, UK},
abstract = {In this paper we focus on event recognition in visual image streams. More specifically, we aim to construct a compact representation which encodes the diversity of the visual stream from just a few observations. For this purpose, we introduce the Event Fisher Vector, a Fisher Kernel based representation to describe a collection of images or the sequential frames of a video. We explore different generative models beyond the Gaussian mixture model as underlying probability distribution. First, the Student's-t mixture model which captures the heavy tails of the small sample size of a collection of images. Second, Hidden Markov Models to explicitly capture the temporal ordering of the observations in a stream. For all our models we derive analytical approximations of the Fisher information matrix, which significantly improves recognition performance. We extensively evaluate the properties of our proposed method on three recent datasets for event recognition in photo collections and web videos, leading to an efficient compact image representation which achieves state-of-the-art performance on all these datasets.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{CappalloICMR15,
title = {Latent Factors of Visual Popularity Prediction},
author = {Spencer Cappallo and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/cappallo-visual-popularity-icmr2015.pdf},
year = {2015},
date = {2015-06-01},
booktitle = {ICMR},
address = {Shanghai, China},
abstract = {Predicting the popularity of an image on social networks based solely on its visual content is a difficult problem. One image may become widely distributed and repeatedly shared, while another similar image may be totally overlooked. We aim to gain insight into how visual content affects image popularity. We propose a latent ranking approach that takes into account not only the distinctive visual cues in popular images, but also those in unpopular images. This method is evaluated on two existing datasets collected from photo-sharing websites, as well as a new proposed dataset of images from the microblogging website Twitter. Our experiments investigate factors of the ranking model, the level of user engagement in scoring popularity, and whether the discovered senses are meaningful. The proposed approach yields state of the art results, and allows for insight into the semantics of image popularity on social networks.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{HabibianICMR15,
title = {Discovering Semantic Vocabularies for Cross-Media Retrieval},
author = {Amirhossein Habibian and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/habibian-semantic-vocabularies-icmr2015.pdf},
year = {2015},
date = {2015-06-01},
booktitle = {ICMR},
address = {Shanghai, China},
abstract = {This paper proposes a data-driven approach for cross-media retrieval by automatically learning its underlying semantic vocabulary. Different from the existing semantic vocabularies, which are manually pre-defined and annotated, we automatically discover the vocabulary concepts and their annotations from multimedia collections. To this end, we apply a probabilistic topic model on the text available in the collection to extractits semantic structure. Moreover, we propose a learning to rank framework, to effectively learn the concept classifiers from the extracted annotations. We evaluate the discovered semantic vocabulary for cross-media retrieval on three datasets of image/text and video/text pairs. Our experiments demonstrate that the discovered vocabulary does not require any manual labeling to outperform three recent alternatives for cross-media retrieval.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{MazloomICMR15,
title = {Encoding Concept Prototypes for Video Event Detection and Summarization},
author = {Masoud Mazloom and Amirhossein Habibian and Dong Liu and Cees G M Snoek and Shih-Fu Chang},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mazloom-concept-prototypes-icmr2015.pdf},
year = {2015},
date = {2015-06-01},
booktitle = {ICMR},
address = {Shanghai, China},
abstract = {This paper proposes a new semantic video representation for few and zero example event detection and unsupervised video event summarization. Different from existing works, which obtain a semantic representation by training concepts over images or entire video clips, we propose an algorithm that learns a set of relevant frames as the concept prototypes from web video examples, without the need for frame-level annotations, and use them for representing an event video. We formulate the problem of learning the concept prototypes as seeking the frames closest to the densest region in the feature space of video frames from both positive and negative training videos of a target concept. We study the behavior of our video event representation based on concept prototypes by performing three experiments on challenging web videos from the TRECVID 2013 multimedia event detection task and the MED-summaries dataset. Our experiments establish that i) Event detection accuracy increases when mapping each video into concept prototype space. ii) Zero-example event detection increases by analyzing each frame of a video individually in concept prototype space, rather than considering the holistic videos. iii) Unsupervised video event summarization using concept prototypes is more accurate than using video-level concept detectors.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{MettesICMR15,
title = {Bag-of-Fragments: Selecting and encoding video fragments for event detection and recounting},
author = {Pascal Mettes and Jan C van Gemert and Spencer Cappallo and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mettes-bag-of-fragments-icmr2015.pdf},
year = {2015},
date = {2015-06-01},
booktitle = {ICMR},
address = {Shanghai, China},
abstract = {The goal of this paper is event detection and recounting using a representation of concept detector scores. Different from existing work, which encodes videos by averaging concept scores over all frames, we propose to encode videos using fragments that are discriminatively learned per event. Our bag-of-fragments split a video into semantically coherent fragment proposals. From training video proposals we show how to select the most discriminative fragment for an event. An encoding of a video is in turn generated by matching and pooling these discriminative fragments to the fragment proposals of the video. The bag-of-fragments forms an effective encoding for event detection and is able to provide a precise temporally localized event recounting. Furthermore, we show how bag-of-fragments can be extended to deal with irrelevant concepts in the event recounting. Experiments on challenging web videos show that i) our modest number of fragment proposals give a high sub-event recall, ii) bag-of-fragments is complementary to global averaging and provides better event detection, iii) bag-of-fragments with concept filtering yields a desirable event recounting. We conclude that fragments matter for video event detection and recounting.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{JainCVPR15,
title = {What do 15,000 object categories tell us about classifying and localizing actions?},
author = {Mihir Jain and Jan C van Gemert and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/jain-objects-actions-cvpr2015.pdf},
year = {2015},
date = {2015-06-01},
booktitle = {CVPR},
address = {Boston, MA, USA},
abstract = {This paper contributes to automatic classification and localization of human actions in video. Whereas motion is the key ingredient in modern approaches, we assess the benefits of having objects in the video representation. Rather than considering a handful of carefully selected and localized objects, we conduct an empirical study on the benefit of encoding 15,000 object categories for action using 6 datasets totaling more than 200 hours of video and covering 180 action classes. Our key contributions are i) the first in-depth study of encoding objects for actions, ii) we show that objects matter for actions, and are often semantically relevant as well. iii) We establish that actions have object preferences. Rather than using all objects, selection is advantageous for action recognition. iv) We reveal that object-action relations are generic, which allows to transferring these relationships from the one domain to the other. And, v) objects, when combined with motion, improve the state-of-the-art for both action classification and localization.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{JainTHUMOS15,
title = {University of Amsterdam at THUMOS Challenge 2015},
author = {Mihir Jain and Jan van Gemert and Pascal Mettes and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/jain-THUMOS2015-final.pdf},
year = {2015},
date = {2015-06-01},
booktitle = {CVPR workshop},
address = {Boston, USA},
abstract = {This notebook paper describes our approach for the action classification task of the THUMOS 2015 benchmark challenge. We use two types of representations to capture motion and appearance. For a local motion description we employ HOG, HOF and MBH features, computed along the improved dense trajectories. The motion features are encoded into a fixed-length representation using Fisher vectors. For the appearance features, we employ a pre-trained GoogLeNet convolutional network on video frames. VLAD is used to encode the appearance features into a fixed-length representation. All actions are classified with a one-vs-rest linear SVM.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{HabibianMM14,
title = {VideoStory: A New Multimedia Embedding for Few-Example Recognition and Translation of Events},
author = {Amirhossein Habibian and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/habibian-videostory-mm2014.pdf},
year = {2014},
date = {2014-11-01},
booktitle = {MM},
pages = {17--26},
address = {Orlando, Florida, USA},
abstract = {This paper proposes a new video representation for few-example event recognition and translation. Different from existing representations, which rely on either low-level features, or pre-specified attributes, we propose to learn an embedding from videos and their descriptions. In our embedding, which we call VideoStory, correlated term labels are combined if their combination improves the video classifier prediction. Our proposed algorithm prevents the combination of correlated terms which are visually dissimilar by optimizing a joint-objective balancing descriptiveness and predictability. The algorithm learns from textual descriptions of video content, which we obtain for free from the web by a simple spidering procedure. We use our VideoStory representation for few-example recognition of events on more than 65K challenging web videos from the NIST TRECVID event detection task and the Columbia Consumer Video collection. Our experiments establish that i) VideoStory outperforms an embedding without joint-objective and alternatives without any embedding, ii) The varying quality of input video descriptions from the web is compensated by harvesting more data, iii) VideoStory sets a new state-of-the-art for few-example event recognition, outperforming very recent attribute and low-level motion encodings. What is more, VideoStory translates a previously unseen video to its most likely description from visual content only.},
note = {Best paper award},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{SnoekTRECVID14,
title = {MediaMill at TRECVID 2014: Searching Concepts, Objects, Instances and Events in Video},
author = {Cees G M Snoek and Koen E A van de Sande and Daniel Fontijne and Spencer Cappallo and Jan van Gemert and Amirhossein Habibian and Thomas Mensink and Pascal Mettes and Ran Tao and Dennis C Koelma and Arnold W M Smeulders},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/mediamill-TRECVID2014-final.pdf},
year = {2014},
date = {2014-11-01},
booktitle = {TRECVID},
address = {Orlando USA},
abstract = {In this paper we summarize our TRECVID 2014 video retrieval experiments. The MediaMill team participated in five tasks: concept detection, object localization, instance search, event recognition and recounting. We experimented with concept detection using deep learning and color difference coding, object localization using FLAIR, instance search by one example, event recognition based on VideoStory, and event recounting using COSTA. Our experiments focus on establishing the video retrieval value of these innovations. The 2014 edition of the TRECVID benchmark has again been a fruitful participation for the MediaMill team, resulting in the best result for concept detection and object localization.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{SesameTRECVID14,
title = {The 2014 SESAME Multimedia Event Detection and Recounting System},
author = {Robert C Bolles and Brian J Burns and James A Herson and Gregory K Myers and Julien van Hout and Wen Wang and Julie Wong and Eric Yeh and Amirhossein Habibian and Dennis C Koelma and Thomas Mensink and Arnold W M Smeulders and Cees G M Snoek and Arnav Agharwal and Song Cao and Kan Chen and Rama Kovvuri and Ram Nevatia and Pramod Sharma},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/Sesame-TRECVID2014-final.pdf},
year = {2014},
date = {2014-11-01},
booktitle = {TRECVID},
address = {Orlando USA},
abstract = {The SESAME (video SEarch with Speed and Accuracy for Multimedia Events) team submitted six runs as a full participant in the Multimedia Event Detection (MED) and Multimedia Event Recounting (MER) evaluations. The SESAME system combines low-level visual, audio, and motion features; high-level semantic concepts for visual objects, scenes, persons, sounds, and actions; automatic speech recognition (ASR); and video optical character recognition (OCR). These three types of features and five types of concepts were used in eight event classifiers. One of the event classifiers, VideoStory, is a new approach that exploits the relationship between semantic concepts and imagery in a large training corpus. The SESAME system uses a total of over 18,000 concepts. We combined the event-detection results for these classifiers using a log-likelihood ratio (LLR) late-fusion method, which uses logistic regression to learn combination weights for event-detection scores from multiple classifiers originating from different data types. The SESAME system generated event recountings based on visual and action concepts, and on concepts recognized by ASR and OCR. Training data included the MED Research dataset, ImageNet, a video dataset from YouTube, the UCF101 and HMDB51 action datasets, the NIST SIN dataset, and Wikipedia. The components that contributed most significantly to event-detection performance were the low- and high-level visual features, low-level motion features, and VideoStory. The LLR late-fusion method significantly improved performance over the best individual classifier for 100Ex and 010Ex. For the Semantic Query (SQ), equal fusion weights, instead of the LLR method, were used in fusion due to the absence of training data.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{LiECCV14,
title = {Attributes Make Sense on Segmented Objects},
author = {Zhenyang Li and Efstratios Gavves and Thomas Mensink and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/li-object-level-attributes-eccv2014.pdf},
year = {2014},
date = {2014-09-01},
booktitle = {ECCV},
address = {Zürich, Switzerland},
abstract = {In this paper we aim for object classification and segmentation by attributes. Where existing work considers attributes either for the global image or for the parts of the object, we propose, as our first novelty, to learn and extract attributes on segments containing the entire object. Object-level attributes suffer less from accidental content around the object and accidental image conditions such as partial occlusions, scale changes and viewpoint changes. As our second novelty, we propose joint learning for simultaneous object classification and segment proposal ranking, solely on the basis of attributes. This naturally brings us to our third novelty: object-level attributes for zero-shot, where we use attribute descriptions of unseen classes for localizing their instances in new images and classifying them accordingly. Results on the Caltech UCSD Birds, Leeds Butterflies, and an a-Pascal subset demonstrate that i) extracting attributes on oracle object-level brings substantial benefits ii) our joint learning model leads to accurate attribute-based classification and segmentation, approaching the oracle results and iii) object-level attributes also allow for zero-shot classification and segmentation. We conclude that attributes make sense on segmented objects.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}

@inproceedings{JainTHUMOS14,
title = {University of Amsterdam at THUMOS Challenge 2014},
author = {Mihir Jain and Jan van Gemert and Cees G M Snoek},
url = {http://isis-data.science.uva.nl/cgmsnoek/pub/jain-THUMOS2014-final.pdf},
year = {2014},
date = {2014-09-01},
booktitle = {ECCV workshop},
address = {Zürich, Switzerland},
abstract = {This notebook paper describes our approach for the action classification task of the THUMOS Challenge 2014. We investigate and exploit the action-object relationship by capturing both motion and related objects. As local descriptors we use HOG, HOF and MBH computed along the improved dense trajectories. For video encoding we rely on Fisher vector. In addition, we employ deep net features learned from object attributes to capture action context. All actions are classified with a one-versus-rest linear SVM.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}