title: multi-thread opening file hangs when fanotify is on
body: <p>I use this fanotify sample to monitor open/access perms on the whole file system(/): <a href="http://git.infradead.org/users/eparis/fanotify-example.git" rel="nofollow">http://git.infradead.org/users/eparis/fanotify-example.git</a>.</p>
  
  <p>Then I have a test program with multiple threads, each thread iterate the sample foder and open/close the files in it, sometimes my program hangs at open().</p>
  
  <p>OS: Ubuntu 2.6.38-11 x86_64.</p>
  
  <p>Is it a bug of fanotify that it does not support multiple-thread opening?</p>
  
  <p>The code of my test program:</p>
  
  <pre><code>#include &lt;stdlib.h&gt;
  #include &lt;stdio.h&gt;
  #include &lt;pthread.h&gt;
  #include &lt;ctype.h&gt;
  #include &lt;unistd.h&gt;
  #include &lt;dirent.h&gt;
  #include &lt;string.h&gt;
  #include &lt;sys/stat.h&gt;
  #include &lt;errno.h&gt;
  #include &lt;fcntl.h&gt;
  #include &lt;dirent.h&gt;
  
  //open file function    
  void open_file( char* file )
  {
      int fd = -1;
      fd = open( file, O_WRONLY, 0x666 );
      if( fd &gt;= 0 )
      {
          printf("open:%s\n", file );
          close( fd );
      }
  }
  
  //iterate directory function
  
  void printdir(char *dir, int depth)
  {
      DIR *dp;
      struct stat statbuf;
      char pathbuf[2048] = {0};
      struct dirent entry;
      struct dirent *entryPtr = NULL;
  
      //printf("opendir %s\n", dir );
      usleep( 300 );
      if((dp = opendir(dir)) == NULL) {
          if( errno != ENOTDIR )
          {
              fprintf(stderr,"cannot open directory: %s\n", dir);
              perror("open fial");
          }
          return;
      }
  
      readdir_r( dp, &amp;entry, &amp;entryPtr );
      while( entryPtr != NULL)
      {
          snprintf(pathbuf,2000, "%s/%s\0", dir, entry.d_name );
          printf("iteraotr:%s\n", pathbuf );
  
          lstat( pathbuf, &amp;statbuf );
          if(S_ISDIR( statbuf.st_mode )) 
          {
              /* Found a directory, but ignore . and .. */
              if(strcmp(".",entry.d_name) == 0 || 
                      strcmp("..",entry.d_name) == 0)
              {
  
              }
              else
              {
                  //printf("%d,%s\n",depth, entry-&gt;d_name);
                  printdir( pathbuf, depth+1);
              }
          }
          else
          {
              //printf("%*s%s\n",depth,"",entry-&gt;d_name);
              open_file( pathbuf );
          }
          readdir_r( dp, &amp;entry, &amp;entryPtr );
      }
      closedir(dp);
  }
  
  //thread function   
  void* iterator_dir( void* data )
  {
      char* path = (char*)data;
      printf("In iterator_dir(): %s\n", path );
  
      printdir( path, 0 );
  
      return NULL;
  }
  
  
  pthread_t  threadID[10] = {0};
  
   //main function    
  int main( int argc, char** argv )
  {
      if( argc &lt; 3 )
      {
          printf("Usage: %s &lt;thread_num&gt; &lt;file&gt;\n", argv[0] );
          exit(0);
      }
      if( isdigit( (char)*argv[1] ) == 0 )
      {
          printf(" Thread num is 0 - 9\n");
          exit(0);
      }
  
      int thread_num = atoi( argv[1] );
      char* res;
  
      pthread_attr_t attr;
      pthread_attr_init(&amp;attr);
      int i = 0;
      for( i = 0; i &lt; thread_num; ++i )
      {
          pthread_create( &amp;threadID[i], &amp;attr, &amp;iterator_dir, argv[2]);
      }
  
      for( i = 0; i &lt; thread_num; ++i )
      {
          pthread_join( threadID[i] , &amp;res );
      }
  }
  </code></pre>
  
  <p><strong>2011-09-28 Edit:</strong></p>
  
  <p>I comment the open file operation, only keep the iterate directory part. The application still hangs.
  This is the output of strace:</p>
  
  <pre><code>enter code here
  pid 10692] open("/home/byang//.config/menus", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC &lt;unfinished ...&gt;
  [pid 10691] write(1, "1213966080 opendir /home/byang//"..., 56) = 56
  </code></pre>
  
  <p>.........</p>
  
  <pre><code>[pid 10689] madvise(0x7f3c48dbc000, 8368128, MADV_DONTNEED) = 0
  [pid 10689] _exit(0)                    = ?
  Process 10689 detached
  [pid 10688] &lt;... futex resumed&gt; )       = 0
  [pid 10688] futex(0x7f3c47db99d0, FUTEX_WAIT, 10692, NULL &lt;unfinished ...&gt;
  </code></pre>
  
  <p>It hangs here, when I close the fanotify, it continues...</p>
  
  <pre><code>[pid 10692] &lt;... open resumed&gt; )        = 11
  [pid 10692] getdents(11, /* 4 entries */, 32768) = 128
  [pid 10692] lstat("/home/byang//.config/menus/applications.menu", {st_mode=S_IFREG|0644, st_size=233, ...}) = 0
  </code></pre>
  
  <p>10688 is the parent thread; 10689,10691,10692 are the child threads that iterating the directories.
  It seems that 10692 are waiting the reply from fanotify?</p>