| 1 | /*
|
|---|
| 2 | * chdev.c - channel device descriptor operations implementation.
|
|---|
| 3 | *
|
|---|
| 4 | * Authors Alain Greiner (2016)
|
|---|
| 5 | *
|
|---|
| 6 | * Copyright (c) UPMC Sorbonne Universites
|
|---|
| 7 | *
|
|---|
| 8 | * This file is part of ALMOS-MKH.
|
|---|
| 9 | *
|
|---|
| 10 | * ALMOS-MKH.is free software; you can redistribute it and/or modify it
|
|---|
| 11 | * under the terms of the GNU General Public License as published by
|
|---|
| 12 | * the Free Software Foundation; version 2.0 of the License.
|
|---|
| 13 | *
|
|---|
| 14 | * ALMOS-MKH.is distributed in the hope that it will be useful, but
|
|---|
| 15 | * WITHOUT ANY WARRANTY; without even the implied warranty of
|
|---|
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|---|
| 17 | * General Public License for more details.
|
|---|
| 18 | *
|
|---|
| 19 | * You should have received a copy of the GNU General Public License
|
|---|
| 20 | * along with ALMOS-MKH.; if not, write to the Free Software Foundation,
|
|---|
| 21 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|---|
| 22 | */
|
|---|
| 23 |
|
|---|
| 24 | #include <kernel_config.h>
|
|---|
| 25 | #include <hal_types.h>
|
|---|
| 26 | #include <hal_special.h>
|
|---|
| 27 | #include <hal_irqmask.h>
|
|---|
| 28 | #include <printk.h>
|
|---|
| 29 | #include <boot_info.h>
|
|---|
| 30 | #include <xlist.h>
|
|---|
| 31 | #include <kmem.h>
|
|---|
| 32 | #include <scheduler.h>
|
|---|
| 33 | #include <thread.h>
|
|---|
| 34 | #include <rpc.h>
|
|---|
| 35 | #include <chdev.h>
|
|---|
| 36 | #include <devfs.h>
|
|---|
| 37 |
|
|---|
| 38 |
|
|---|
| 39 | extern chdev_directory_t chdev_dir; // allocated in kernel_init.c
|
|---|
| 40 |
|
|---|
| 41 | #if CONFIG_READ_DEBUG
|
|---|
| 42 | extern uint32_t enter_chdev_cmd;
|
|---|
| 43 | extern uint32_t exit_chdev_cmd;
|
|---|
| 44 | extern uint32_t enter_chdev_server;
|
|---|
| 45 | extern uint32_t exit_chdev_server;
|
|---|
| 46 | #endif
|
|---|
| 47 |
|
|---|
| 48 | ////////////////////////////////////////////
|
|---|
| 49 | char * chdev_func_str( uint32_t func_type )
|
|---|
| 50 | {
|
|---|
| 51 | if ( func_type == DEV_FUNC_RAM ) return "RAM";
|
|---|
| 52 | else if( func_type == DEV_FUNC_ROM ) return "ROM";
|
|---|
| 53 | else if( func_type == DEV_FUNC_FBF ) return "FBF";
|
|---|
| 54 | else if( func_type == DEV_FUNC_IOB ) return "IOB";
|
|---|
| 55 | else if( func_type == DEV_FUNC_IOC ) return "IOC";
|
|---|
| 56 | else if( func_type == DEV_FUNC_MMC ) return "MMC";
|
|---|
| 57 | else if( func_type == DEV_FUNC_DMA ) return "DMA";
|
|---|
| 58 | else if( func_type == DEV_FUNC_NIC ) return "NIC";
|
|---|
| 59 | else if( func_type == DEV_FUNC_TIM ) return "TIM";
|
|---|
| 60 | else if( func_type == DEV_FUNC_TXT ) return "TXT";
|
|---|
| 61 | else if( func_type == DEV_FUNC_ICU ) return "ICU";
|
|---|
| 62 | else if( func_type == DEV_FUNC_PIC ) return "PIC";
|
|---|
| 63 | else return "undefined";
|
|---|
| 64 | }
|
|---|
| 65 |
|
|---|
| 66 | /////////////////////////////////////////
|
|---|
| 67 | chdev_t * chdev_create( uint32_t func,
|
|---|
| 68 | uint32_t impl,
|
|---|
| 69 | uint32_t channel,
|
|---|
| 70 | uint32_t is_rx,
|
|---|
| 71 | xptr_t base )
|
|---|
| 72 | {
|
|---|
| 73 | chdev_t * chdev;
|
|---|
| 74 | kmem_req_t req;
|
|---|
| 75 |
|
|---|
| 76 | // allocate memory for chdev
|
|---|
| 77 | req.type = KMEM_DEVICE;
|
|---|
| 78 | req.flags = AF_ZERO;
|
|---|
| 79 | chdev = (chdev_t *)kmem_alloc( &req );
|
|---|
| 80 |
|
|---|
| 81 | if( chdev == NULL ) return NULL;
|
|---|
| 82 |
|
|---|
| 83 | // initialize waiting threads queue and associated lock
|
|---|
| 84 | remote_spinlock_init( XPTR( local_cxy , &chdev->wait_lock ) );
|
|---|
| 85 | xlist_root_init( XPTR( local_cxy , &chdev->wait_root ) );
|
|---|
| 86 |
|
|---|
| 87 | // initialize attributes
|
|---|
| 88 | chdev->func = func;
|
|---|
| 89 | chdev->impl = impl;
|
|---|
| 90 | chdev->channel = channel;
|
|---|
| 91 | chdev->is_rx = is_rx;
|
|---|
| 92 | chdev->base = base;
|
|---|
| 93 |
|
|---|
| 94 | return chdev;
|
|---|
| 95 |
|
|---|
| 96 | } // end chdev_create()
|
|---|
| 97 |
|
|---|
| 98 | ///////////////////////////////////
|
|---|
| 99 | void chdev_print( chdev_t * chdev )
|
|---|
| 100 | {
|
|---|
| 101 | printk("\n - func = %s"
|
|---|
| 102 | "\n - channel = %d"
|
|---|
| 103 | "\n - base = %l"
|
|---|
| 104 | "\n - cmd = %x"
|
|---|
| 105 | "\n - isr = %x"
|
|---|
| 106 | "\n - chdev = %x\n",
|
|---|
| 107 | chdev_func_str(chdev->func),
|
|---|
| 108 | chdev->channel,
|
|---|
| 109 | chdev->base,
|
|---|
| 110 | chdev->cmd,
|
|---|
| 111 | chdev->isr,
|
|---|
| 112 | chdev );
|
|---|
| 113 | }
|
|---|
| 114 |
|
|---|
| 115 | //////////////////////////////////////////////////
|
|---|
| 116 | void chdev_register_command( xptr_t chdev_xp )
|
|---|
| 117 | {
|
|---|
| 118 | thread_t * server_ptr; // local pointer on server thread associated to chdev
|
|---|
| 119 | core_t * core_ptr; // local pointer on core running the server thread
|
|---|
| 120 | uint32_t lid; // core running the server thread local index
|
|---|
| 121 | xptr_t lock_xp; // extended pointer on lock protecting the chdev queue
|
|---|
| 122 | uint32_t different; // non zero if server thread core != client thread core
|
|---|
| 123 | uint32_t save_sr; // for critical section
|
|---|
| 124 |
|
|---|
| 125 | thread_t * this = CURRENT_THREAD;
|
|---|
| 126 |
|
|---|
| 127 | chdev_dmsg("\n[DBG] %s : core[%x,%d] (thread %s) enter / cycle %d\n",
|
|---|
| 128 | __FUNCTION__, local_cxy, this->core->lid, thread_type_str(this->type) , hal_time_stamp() );
|
|---|
| 129 |
|
|---|
| 130 | // get device descriptor cluster and local pointer
|
|---|
| 131 | cxy_t chdev_cxy = GET_CXY( chdev_xp );
|
|---|
| 132 | chdev_t * chdev_ptr = (chdev_t *)GET_PTR( chdev_xp );
|
|---|
| 133 |
|
|---|
| 134 | // build extended pointers on client thread xlist and device root
|
|---|
| 135 | xptr_t list_xp = XPTR( local_cxy , &this->wait_list );
|
|---|
| 136 | xptr_t root_xp = XPTR( chdev_cxy , &chdev_ptr->wait_root );
|
|---|
| 137 |
|
|---|
| 138 | // get local pointer on server thread
|
|---|
| 139 | server_ptr = (thread_t *)hal_remote_lpt( XPTR( chdev_cxy , &chdev_ptr->server) );
|
|---|
| 140 |
|
|---|
| 141 | chdev_dmsg("\n[DBG] %s : core[%x,%d] (thread %s) / server_cxy %x / server_ptr %x / server_type %\n",
|
|---|
| 142 | __FUNCTION__, local_cxy, this->core->lid, server_cxy, server_ptr,
|
|---|
| 143 | thread_type_str( hal_remote_lw( XPTR( server_cxy , &server_ptr->type) ) ) );
|
|---|
| 144 |
|
|---|
| 145 | // build extended pointer on chdev lock protecting queue
|
|---|
| 146 | lock_xp = XPTR( chdev_cxy , &chdev_ptr->wait_lock );
|
|---|
| 147 |
|
|---|
| 148 | // get local pointer on core running the server thread
|
|---|
| 149 | core_ptr = (core_t *)hal_remote_lpt( XPTR( chdev_cxy , &server_ptr->core ) );
|
|---|
| 150 |
|
|---|
| 151 | // get core local index
|
|---|
| 152 | lid = hal_remote_lw( XPTR( chdev_cxy , &core_ptr->lid ) );
|
|---|
| 153 |
|
|---|
| 154 | // compute server core != thread core
|
|---|
| 155 | different = (lid != this->core->lid) || (local_cxy != chdev_cxy);
|
|---|
| 156 |
|
|---|
| 157 | // enter critical section to make atomic :
|
|---|
| 158 | // (1) client blocking
|
|---|
| 159 | // (2) client registration in server queue
|
|---|
| 160 | // (3) IPI to force server scheduling
|
|---|
| 161 | // (4) descheduling
|
|---|
| 162 | // ... in this order
|
|---|
| 163 | hal_disable_irq( &save_sr );
|
|---|
| 164 |
|
|---|
| 165 | // block current thread
|
|---|
| 166 | thread_block( CURRENT_THREAD , THREAD_BLOCKED_IO );
|
|---|
| 167 |
|
|---|
| 168 | // register client thread in waiting queue
|
|---|
| 169 | remote_spinlock_lock( lock_xp );
|
|---|
| 170 | xlist_add_last( root_xp , list_xp );
|
|---|
| 171 | remote_spinlock_unlock( lock_xp );
|
|---|
| 172 |
|
|---|
| 173 | // send IPI to core running the server thread if required
|
|---|
| 174 | if( different ) dev_pic_send_ipi( chdev_cxy , lid );
|
|---|
| 175 |
|
|---|
| 176 | chdev_dmsg("\n[DBG] %s : core[%x,%d] (thread %s) deschedules / cycle %d\n",
|
|---|
| 177 | __FUNCTION__, local_cxy, this->core->lid, thread_type_str(this->type) , hal_time_stamp() );
|
|---|
| 178 |
|
|---|
| 179 | // deschedule
|
|---|
| 180 | assert( thread_can_yield( this ) , __FUNCTION__ , "illegal sched_yield\n" );
|
|---|
| 181 | sched_yield("blocked on I/O");
|
|---|
| 182 |
|
|---|
| 183 | chdev_dmsg("\n[DBG] %s : core[%x,%d] (thread %s) resumes / cycle %d\n",
|
|---|
| 184 | __FUNCTION__, local_cxy, this->core->lid, thread_type_str(this->type) , hal_time_stamp() );
|
|---|
| 185 |
|
|---|
| 186 | // exit critical section
|
|---|
| 187 | hal_restore_irq( save_sr );
|
|---|
| 188 |
|
|---|
| 189 | } // end chdev_register_command()
|
|---|
| 190 |
|
|---|
| 191 | ///////////////////////////////////////////////
|
|---|
| 192 | void chdev_sequencial_server( chdev_t * chdev )
|
|---|
| 193 | {
|
|---|
| 194 | xptr_t client_xp; // extended pointer on waiting thread
|
|---|
| 195 | cxy_t client_cxy; // cluster of client thread
|
|---|
| 196 | thread_t * client_ptr; // local pointer on client thread
|
|---|
| 197 | thread_t * server; // local pointer on server thread
|
|---|
| 198 | xptr_t root_xp; // extended pointer on device waiting queue root
|
|---|
| 199 | xptr_t lock_xp; // extended pointer on lock ptotecting chdev queue
|
|---|
| 200 |
|
|---|
| 201 | server = CURRENT_THREAD;
|
|---|
| 202 |
|
|---|
| 203 | chdev_dmsg("\n[DBG] %s : enter / server = %x / chdev = %x / cycle %d\n",
|
|---|
| 204 | __FUNCTION__ , server , chdev , hal_time_stamp() );
|
|---|
| 205 |
|
|---|
| 206 | root_xp = XPTR( local_cxy , &chdev->wait_root );
|
|---|
| 207 | lock_xp = XPTR( local_cxy , &chdev->wait_lock );
|
|---|
| 208 |
|
|---|
| 209 | // This infinite loop is executed by the DEV thread
|
|---|
| 210 | // to handle commands registered in the chdev queue.
|
|---|
| 211 | while( 1 )
|
|---|
| 212 | {
|
|---|
| 213 | // get the lock protecting the waiting queue
|
|---|
| 214 | remote_spinlock_lock( lock_xp );
|
|---|
| 215 |
|
|---|
| 216 | // check waiting queue state
|
|---|
| 217 | if( xlist_is_empty( root_xp ) ) // waiting queue empty
|
|---|
| 218 | {
|
|---|
| 219 | // release lock
|
|---|
| 220 | remote_spinlock_unlock( lock_xp );
|
|---|
| 221 |
|
|---|
| 222 | chdev_dmsg("\n[DBG] %s : thread %x deschedule /cycle %d\n",
|
|---|
| 223 | __FUNCTION__ , server , hal_time_stamp() );
|
|---|
| 224 |
|
|---|
| 225 | // deschedule
|
|---|
| 226 | sched_yield("I/O queue empty");
|
|---|
| 227 |
|
|---|
| 228 | chdev_dmsg("\n[DBG] %s : thread %x resume /cycle %d\n",
|
|---|
| 229 | __FUNCTION__ , server , hal_time_stamp() );
|
|---|
| 230 |
|
|---|
| 231 | }
|
|---|
| 232 | else // waiting queue not empty
|
|---|
| 233 | {
|
|---|
| 234 |
|
|---|
| 235 | #if CONFIG_READ_DEBUG
|
|---|
| 236 | enter_chdev_server = hal_time_stamp();
|
|---|
| 237 | #endif
|
|---|
| 238 | // release lock
|
|---|
| 239 | remote_spinlock_unlock( lock_xp );
|
|---|
| 240 |
|
|---|
| 241 | // get extended pointer on first client thread
|
|---|
| 242 | client_xp = XLIST_FIRST_ELEMENT( root_xp , thread_t , wait_list );
|
|---|
| 243 |
|
|---|
| 244 | // get client thread cluster, local pointer, and identifier
|
|---|
| 245 | client_cxy = GET_CXY( client_xp );
|
|---|
| 246 | client_ptr = (thread_t *)GET_PTR( client_xp );
|
|---|
| 247 |
|
|---|
| 248 | // call driver command function to execute I/O operation
|
|---|
| 249 | chdev->cmd( client_xp );
|
|---|
| 250 |
|
|---|
| 251 | // remove the client thread from waiting queue
|
|---|
| 252 | remote_spinlock_lock( lock_xp );
|
|---|
| 253 | xlist_unlink( XPTR( client_cxy , &client_ptr->wait_list ) );
|
|---|
| 254 | remote_spinlock_unlock( lock_xp );
|
|---|
| 255 |
|
|---|
| 256 | chdev_dmsg("\n[DBG] %s : thread %x complete operation for client %x / cycle %d\n",
|
|---|
| 257 | __FUNCTION__ , server , client_ptr , hal_time_stamp() );
|
|---|
| 258 |
|
|---|
| 259 | #if CONFIG_READ_DEBUG
|
|---|
| 260 | exit_chdev_server = hal_time_stamp();
|
|---|
| 261 | #endif
|
|---|
| 262 |
|
|---|
| 263 | }
|
|---|
| 264 | } // end while
|
|---|
| 265 | } // end chdev_sequencial_server()
|
|---|
| 266 |
|
|---|
| 267 | ////////////////////////
|
|---|
| 268 | void chdev_dir_display()
|
|---|
| 269 | {
|
|---|
| 270 | cxy_t iob_cxy = GET_CXY( chdev_dir.iob );
|
|---|
| 271 | chdev_t * iob_ptr = (chdev_t *)GET_PTR( chdev_dir.iob );
|
|---|
| 272 | uint32_t iob_base = (uint32_t)hal_remote_lwd( XPTR( iob_cxy , &iob_ptr->base ) );
|
|---|
| 273 |
|
|---|
| 274 | cxy_t pic_cxy = GET_CXY( chdev_dir.pic );
|
|---|
| 275 | chdev_t * pic_ptr = (chdev_t *)GET_PTR( chdev_dir.pic );
|
|---|
| 276 | uint32_t pic_base = (uint32_t)hal_remote_lwd( XPTR( pic_cxy , &pic_ptr->base ) );
|
|---|
| 277 |
|
|---|
| 278 | cxy_t txt0_tx_cxy = GET_CXY( chdev_dir.txt_tx[0] );
|
|---|
| 279 | chdev_t * txt0_tx_ptr = (chdev_t *)GET_PTR( chdev_dir.txt_tx[0] );
|
|---|
| 280 | uint32_t txt0_tx_base = (uint32_t)hal_remote_lwd( XPTR( txt0_tx_cxy , &txt0_tx_ptr->base ) );
|
|---|
| 281 |
|
|---|
| 282 | cxy_t txt0_rx_cxy = GET_CXY( chdev_dir.txt_rx[0] );
|
|---|
| 283 | chdev_t * txt0_rx_ptr = (chdev_t *)GET_PTR( chdev_dir.txt_rx[0] );
|
|---|
| 284 | uint32_t txt0_rx_base = (uint32_t)hal_remote_lwd( XPTR( txt0_rx_cxy , &txt0_rx_ptr->base ) );
|
|---|
| 285 |
|
|---|
| 286 | cxy_t txt1_tx_cxy = GET_CXY( chdev_dir.txt_tx[1] );
|
|---|
| 287 | chdev_t * txt1_tx_ptr = (chdev_t *)GET_PTR( chdev_dir.txt_tx[1] );
|
|---|
| 288 | uint32_t txt1_tx_base = (uint32_t)hal_remote_lwd( XPTR( txt1_tx_cxy , &txt1_tx_ptr->base ) );
|
|---|
| 289 |
|
|---|
| 290 | cxy_t txt1_rx_cxy = GET_CXY( chdev_dir.txt_rx[1] );
|
|---|
| 291 | chdev_t * txt1_rx_ptr = (chdev_t *)GET_PTR( chdev_dir.txt_rx[1] );
|
|---|
| 292 | uint32_t txt1_rx_base = (uint32_t)hal_remote_lwd( XPTR( txt1_rx_cxy , &txt1_rx_ptr->base ) );
|
|---|
| 293 |
|
|---|
| 294 | cxy_t txt2_tx_cxy = GET_CXY( chdev_dir.txt_tx[2] );
|
|---|
| 295 | chdev_t * txt2_tx_ptr = (chdev_t *)GET_PTR( chdev_dir.txt_tx[2] );
|
|---|
| 296 | uint32_t txt2_tx_base = (uint32_t)hal_remote_lwd( XPTR( txt2_tx_cxy , &txt2_tx_ptr->base ) );
|
|---|
| 297 |
|
|---|
| 298 | cxy_t txt2_rx_cxy = GET_CXY( chdev_dir.txt_rx[2] );
|
|---|
| 299 | chdev_t * txt2_rx_ptr = (chdev_t *)GET_PTR( chdev_dir.txt_rx[2] );
|
|---|
| 300 | uint32_t txt2_rx_base = (uint32_t)hal_remote_lwd( XPTR( txt2_rx_cxy , &txt2_rx_ptr->base ) );
|
|---|
| 301 |
|
|---|
| 302 | cxy_t ioc_cxy = GET_CXY( chdev_dir.ioc[0] );
|
|---|
| 303 | chdev_t * ioc_ptr = (chdev_t *)GET_PTR( chdev_dir.ioc[0] );
|
|---|
| 304 | uint32_t ioc_base = (uint32_t)hal_remote_lwd( XPTR( ioc_cxy , &ioc_ptr->base ) );
|
|---|
| 305 |
|
|---|
| 306 | cxy_t fbf_cxy = GET_CXY( chdev_dir.fbf[0] );
|
|---|
| 307 | chdev_t * fbf_ptr = (chdev_t *)GET_PTR( chdev_dir.fbf[0] );
|
|---|
| 308 | uint32_t fbf_base = (uint32_t)hal_remote_lwd( XPTR( fbf_cxy , &fbf_ptr->base ) );
|
|---|
| 309 |
|
|---|
| 310 | cxy_t nic0_rx_cxy = GET_CXY( chdev_dir.nic_rx[0] );
|
|---|
| 311 | chdev_t * nic0_rx_ptr = (chdev_t *)GET_PTR( chdev_dir.nic_rx[0] );
|
|---|
| 312 | uint32_t nic0_rx_base = (uint32_t)hal_remote_lwd( XPTR( nic0_rx_cxy , &nic0_rx_ptr->base ) );
|
|---|
| 313 |
|
|---|
| 314 | cxy_t nic0_tx_cxy = GET_CXY( chdev_dir.nic_tx[0] );
|
|---|
| 315 | chdev_t * nic0_tx_ptr = (chdev_t *)GET_PTR( chdev_dir.nic_tx[0] );
|
|---|
| 316 | uint32_t nic0_tx_base = (uint32_t)hal_remote_lwd( XPTR( nic0_tx_cxy , &nic0_tx_ptr->base ) );
|
|---|
| 317 |
|
|---|
| 318 | printk("\n***** external chdev directory in cluster %x\n"
|
|---|
| 319 | " - iob : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 320 | " - pic : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 321 | " - ioc : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 322 | " - fbf : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 323 | " - txt_rx[0] : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 324 | " - txt_tx[0] : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 325 | " - txt_rx[1] : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 326 | " - txt_tx[1] : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 327 | " - txt_rx[2] : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 328 | " - txt_tx[2] : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 329 | " - nic_rx[0] : cxy = %X / ptr = %X / base = %X\n"
|
|---|
| 330 | " - nic_tx[0] : cxy = %X / ptr = %X / base = %X\n",
|
|---|
| 331 | local_cxy,
|
|---|
| 332 | iob_cxy , iob_ptr , iob_base ,
|
|---|
| 333 | pic_cxy , pic_ptr , pic_base ,
|
|---|
| 334 | ioc_cxy , ioc_ptr , ioc_base ,
|
|---|
| 335 | fbf_cxy , fbf_ptr , fbf_base ,
|
|---|
| 336 | txt0_rx_cxy , txt0_rx_ptr , txt0_rx_base ,
|
|---|
| 337 | txt0_tx_cxy , txt0_tx_ptr , txt0_tx_base ,
|
|---|
| 338 | txt1_rx_cxy , txt1_rx_ptr , txt1_rx_base ,
|
|---|
| 339 | txt1_tx_cxy , txt1_tx_ptr , txt1_tx_base ,
|
|---|
| 340 | txt2_rx_cxy , txt2_rx_ptr , txt2_rx_base ,
|
|---|
| 341 | txt2_tx_cxy , txt2_tx_ptr , txt2_tx_base ,
|
|---|
| 342 | nic0_rx_cxy , nic0_rx_ptr , nic0_rx_base ,
|
|---|
| 343 | nic0_tx_cxy , nic0_tx_ptr , nic0_tx_base );
|
|---|
| 344 |
|
|---|
| 345 | } // end chdev_dir_display()
|
|---|
| 346 |
|
|---|