Publications

@techreport{Marques2021,

title = {Multi-scale hierarchical neural network models that bridge from single neurons in the primate primary visual cortex to object recognition behavior},

author = {Tiago Marques and Martin Schrimpf and James J DiCarlo},

url = {https://www.biorxiv.org/content/10.1101/2021.03.01.433495v2 https://www.biorxiv.org/content/10.1101/2021.03.01.433495v2.abstract https://www.biorxiv.org/content/10.1101/2021.03.01.433495v1},

doi = {10.1101/2021.03.01.433495},

year  = {2021},

date = {2021-08-01},

urldate = {2021-08-01},

journal = {bioRxiv preprint},

publisher = {Cold Spring Harbor Laboratory},

abstract = {Object recognition relies on inferior temporal (IT) cortical neural population representations that are themselves computed by a hierarchical network of feedforward and recurrently connected neural population called the ventral visual stream (areas V1, V2, V4 and IT). While recent work has created some reasonably accurate image-computable hierarchical neural network models of those neural stages, those models do not yet bridge between the properties of individual neurons and the overall emergent behavior of the ventral stream. For example, current leading ventral stream models do not allow us to ask questions such as: How does the surround suppression behavior of individual V1 neurons ultimately relate to IT neural representation and to behavior?; or How would deactivation of a particular sub-population of V1 neurons specifically alter object recognition behavior? One reason we cannot yet do this is that individual V1 artificial neurons in multi-stage models have not been shown to be functionally similar with individual biological V1 neurons. Here, we took an important first step towards this direction by building and evaluating hundreds of hierarchical neural network models in how well their artificial single neurons approximate macaque primary visual cortical (V1) neurons. We found that single neurons in some models are surprisingly similar to their biological counterparts and that the distributions of single neuron properties, such as those related to orientation and spatial frequency tuning, approximately match those in macaque V1. Crucially, we also observed that hierarchical models with V1-layers that better match macaque V1 at the single neuron level are also more aligned with human object recognition behavior. These results provide the first multi-stage, multi-scale models that allow our field to ask precisely how the specific properties of individual V1 neurons relate to recognition behavior. Finally, we here show that an optimized classical neuroscientific model of V1 is still more functionally similar to primate V1 than all of the tested multi-stage models, suggesting that further model improvements are possible, and that those improvements would likely have tangible payoffs in terms of behavioral prediction accuracy and behavioral robustness. ### Competing Interest Statement The authors have declared no competing interest.},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{GeigerSchrimpf2020,

title = {Wiring Up Vision : Minimizing Supervised Synaptic Updates Needed to Produce a Primate Ventral Stream},

author = {Franziska Geiger and Martin Schrimpf and Tiago Marques and James J Dicarlo},

url = {https://www.biorxiv.org/content/10.1101/2020.06.08.140111v1},

doi = {10.1101/2020.06.08.140111},

year  = {2020},

date = {2020-06-01},

journal = {bioRxiv preprint},

publisher = {Cold Spring Harbor Laboratory},

abstract = {After training on large datasets, certain deep neural networks are surprisingly good models of the neural mechanisms of adult primate visual object recognition. Nevertheless, these models are poor models of the development of the visual system because they posit millions of sequential, precisely coordinated synaptic updates, each based on a labeled image. While ongoing research is pursuing the use of unsupervised proxies for labels, we here explore a complementary strategy of reducing the required number of supervised synaptic updates to produce an adult-like ventral visual stream (as judged by the match to V1, V2, V4, IT, and behavior). Such models might require less precise machinery and energy expenditure to coordinate these updates and would thus move us closer to viable neuroscientific hypotheses about how the visual system wires itself up. Relative to the current leading model of the adult ventral stream, we here demonstrate that the total number of supervised weight updates can be substantially reduced using three complementary strategies: First, we find that only 2% of supervised updates (epochs and images) are needed to achieve ~80% of the match to adult ventral stream. Second, by improving the random distribution of synaptic connectivity, we find that 54% of the brain match can already be achieved “at birth” (i.e. no training at all). Third, we find that, by training only ~5% of model synapses, we can still achieve nearly 80% of the match to the ventral stream. When these three strategies are applied in combination, we find that these new models achieve ~80% of a fully trained model's match to the brain, while using two orders of magnitude fewer supervised synaptic updates. These results reflect first steps in modeling not just primate adult visual processing during inference, but also how the ventral visual stream might be “wired up” by evolution (a model's “birth” state) and by developmental learning (a model's updates based on visual experience). ### Competing Interest Statement The authors have declared no competing interest.},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{Jozwik2019primatehuman,

title = {To find better neural network models of human vision, find better neural network models of primate vision},

author = {Kamila Maria Jozwik and Martin Schrimpf and Nancy Kanwisher and James J. DiCarlo},

url = {https://www.biorxiv.org/content/10.1101/688390v1.full https://www.biorxiv.org/content/10.1101/688390v1},

doi = {10.1101/688390},

year  = {2019},

date = {2019-07-01},

journal = {bioRxiv},

publisher = {Cold Spring Harbor Laboratory},

abstract = {Specific deep artificial neural networks (ANNs) are the current best models of ventral visual processing and object recognition behavior in monkeys. We here explore whether models of non-human primate vision generalize to visual processing in the human primate brain. Specifically, we asked if model match to monkey IT is a predictor of model match to human IT, even when scoring those matches on different images. We found that the model match to monkey IT is a positive predictor of the model match to human IT (R = 0.36), and that this approach outperforms the current standard predictor of model accuracy on ImageNet. This suggests a more powerful approach for pre-selecting models as hypotheses of human brain processing.},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{KubiliusSchrimpf2018CORnet,

title = {CORnet: Modeling the Neural Mechanisms of Core Object Recognition},

author = {Jonas Kubilius and Martin Schrimpf and Aran Nayebi and Daniel Bear and Daniel L K Yamins and James J Dicarlo},

url = {https://www.biorxiv.org/content/early/2018/09/04/408385 https://www.biorxiv.org/content/biorxiv/early/2018/09/04/408385.full.pdf?%3Fcollection=},

doi = {10.1101/408385},

year  = {2018},

date = {2018-09-01},

journal = {bioRxiv},

publisher = {Cold Spring Harbor Laboratory},

abstract = {Deep artificial neural networks with spatially repeated processing (a.k.a., deep convolutional ANNs) have been established as the best class of candidate models of visual processing in primate ventral visual processing stream. Over the past five years, these ANNs have evolved from a simple feedforward eight-layer architecture in AlexNet to extremely deep and branching NAS-Net architectures, demonstrating increasingly better object cat-egorization performance and increasingly better explanatory power of both neural and behavioral responses. However, from the neuroscientist's point of view, the relationship between such very deep architectures and the ventral visual pathway is incomplete in at least two ways. On the one hand, current state-of-the-art ANNs appear to be too complex (e.g., now over 100 levels) compared with the relatively shallow cortical hierarchy (4-8 levels), which makes it difficult to map their elements to those in the ventral visual stream and to understand what they are doing. On the other hand, current state-of-the-art ANNs appear to be not complex enough in that they lack recurrent connections and the resulting neural response dynamics that are commonplace in the ventral visual stream. Here we describe our ongoing efforts to resolve both of these issues by developing a "CORnet" family of deep neural network architectures. Rather than just seeking high object recognition performance (as the state-of-the-art ANNs above), we instead try to reduce the model family to its most important elements and then gradually build new ANNs with recurrent and skip connections while monitoring both performance and the match between each new CORnet model and a large body of primate brain and behav-ioral data. We report here that our current best ANN model derived from this approach (CORnet-S) is among the top models on Brain-Score, a composite benchmark for comparing models to the brain, but is simpler than other deep ANNs in terms of the number of convolutions performed along the longest path of information processing in the model. All CORnet models are available at github.com/dicarlolab/CORnet, and we plan to update this manuscript and the available models in this family as they are produced. object recognition | deep neural networks | feedforward | recurrence},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{SchrimpfKubilius2018BrainScore,

title = {Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?},

author = {Martin Schrimpf and Jonas Kubilius and Ha Hong and Najib J Majaj and Rishi Rajalingham and Elias B Issa and Kohitij Kar and Pouya Bashivan and Jonathan Prescott-Roy and Kailyn Schmidt and Daniel L K Yamins and James J Dicarlo},

url = {https://www.biorxiv.org/content/early/2018/09/05/407007 http://dx.doi.org/10.1101/407007},

doi = {10.1101/407007},

year  = {2018},

date = {2018-09-01},

journal = {bioRxiv},

publisher = {Cold Spring Harbor Laboratory},

abstract = {The internal representations of early deep artificial neural networks (ANNs) were found to be remarkably similar to the internal neural representations measured experimentally in the primate brain. Here we ask, as deep ANNs have continued to evolve, are they becoming more or less brain-like? ANNs that are most functionally similar to the brain will contain mechanisms that are most like those used by the brain. We therefore developed Brain-Score-a composite of multiple neural and be-havioral benchmarks that score any ANN on how similar it is to the brain's mechanisms for core object recognition-and we deployed it to evaluate a wide range of state-of-the-art deep ANNs. Using this scoring system, we here report that: (1) DenseNet-169, CORnetS and ResNet-101 are the most brain-like ANNs. (2) There remains considerable variability in neural and behav-ioral responses that is not predicted by any ANN, suggesting that no ANN model has yet captured all the relevant mechanisms. (3) Extending prior work, we found that gains in ANN ImageNet performance led to gains on Brain-Score. However, correlation weakened at ≥ 70% top-1 ImageNet performance, suggesting that additional guidance from neuroscience is needed to make further advances in capturing brain mechanisms. (4) We uncovered smaller (i.e. less complex) ANNs that are more brain-like than many of the best-performing ImageNet models, which suggests the opportunity to simplify ANNs to better understand the ventral stream. The scoring system used here is far from complete. However, we propose that evaluating and tracking model-benchmark correspondences through a Brain-Score that is regularly updated with new brain data is an exciting opportunity: experimental benchmarks can be used to guide machine network evolution, and machine networks are mechanistic hypotheses of the brain's network and thus drive next experiments. To facilitate both of these, we release Brain-Score.org: a platform that hosts the neural and behavioral benchmarks, where ANNs for visual processing can be submitted to receive a Brain-Score and their rank relative to other models, and where new experimental data can be naturally incorporated.},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{arend2018single,

title = {Single units in a deep neural network functionally correspond with neurons in the brain: preliminary results},

author = {Luke Arend and Yena Han and Martin Schrimpf and Pouya Bashivan and Kohitij Kar and Tomaso Poggio and James J DiCarlo and Xavier Boix},

year  = {2018},

date = {2018-01-01},

journal = {CBMM Memo},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{cheney2017,

title = {On the Robustness of Convolutional Neural Networks to Internal Architecture and Weight Perturbations},

author = {Nicholas Cheney* and Martin Schrimpf* and Gabriel Kreiman},

url = {https://arxiv.org/abs/1703.08245},

year  = {2017},

date = {2017-03-23},

journal = {arXiv preprint},

abstract = {Deep convolutional neural networks are generally regarded as robust function approximators. So far, this intuition is based on perturbations to external stimuli such as the images to be classified. Here we explore the robustness of convolutional neural networks to perturbations to the internal weights and architecture of the network itself. We show that convolutional networks are surprisingly robust to a number of internal perturbations in the higher convolutional layers but the bottom convolutional layers are much more fragile. For instance, Alexnet shows less than a 30% decrease in classification performance when randomly removing over 70% of weight connections in the top convolutional or dense layers but performance is almost at chance with the same perturbation in the first convolutional layer. Finally, we suggest further investigations which could continue to inform the robustness of convolutional networks to internal perturbations.},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{bachelors_thesisb,

title = {Scalable Database Concurrency Control using Transactional Memory},

author = {Martin Schrimpf},

url = {http://mschrimpf.com/wp-content/uploads/2016/11/Scalable-Database-Concurrency-Control-using-Transactional-Memory-Martin-Schrimpf-TextSigned.pdf},

year  = {2014},

date = {2014-01-01},

institution = {Technical University Munich},

abstract = {Intel recently made available the optimistic synchronization technique Hardware Transactional 

Memory (HTM) in their mainstream Haswell processor microarchitecture. 

The first part of this work evaluates the core performance characteristics of the two programming 

interfaces within Intel’s Transactional Synchronization Extensions (TSX), Hardware Lock Elision 

(HLE) and Restricted Transactional Memory (RTM). Therein, a scope of application is defined 

regarding inter alia the transaction size which is limited to the L1 DCache or even less with wrongly 

aligned data due to cache associativity, the transaction duration restricted by Hardware interrupts 

and a limit to the nesting of transactions. By comparing common data structures and analyzing 

the behavior of HTM using hardware counters, the Hashmap is identified as a suitable structure 

with a 134% speedup compared to classical POSIX mutexes. 

In the second part, several latching mechanisms of MySQL InnoDB’s Concurrency Control are 

selected and modified with different implementations of HTM to achieve increased scalability. We 

find that it does not suffice to apply HTM naively to all mutex calls by using either HLE prefixes or 

an HTM-enabled glibc. Furthermore, many transactional cycles often come at the price of frequent 

aborted cycles which inhibits performance increases when measuring MySQL with the tx-bench 

and too many aborts can even decrease the throughput to 29% of the unmodified version.},

type = {Bachelor Thesis},

keywords = {},

pubstate = {published},

tppubtype = {techreport}

}