Browse Source

add hbase configuration

ZhenQin 12 years ago
parent
commit
9615d437db
3 changed files with 518 additions and 0 deletions
  1. 118 0
      hbase/hbase-env.sh
  2. 196 0
      hbase/hbase-site.xml
  3. 204 0
      nginx/nginx.conf

+ 118 - 0
hbase/hbase-env.sh

@@ -0,0 +1,118 @@
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set environment variables here.
+
+# This script sets variables multiple times over the course of starting an hbase process,
+# so try to keep things idempotent unless you want to take an even deeper look
+# into the startup scripts (bin/hbase, etc.)
+
+# The java implementation to use.  Java 1.6 required.
+export JAVA_HOME=/opt/jdk1.6.0_45
+
+# Extra Java CLASSPATH elements.  Optional.
+# export HBASE_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default.  May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+export HBASE_MASTER_OPTS=" -Xmx1G -Xms1G "
+export HBASE_REGIONSERVER_OPTS=" -Xmx8G -Xms8G -XX:+CMSIncrementalMode "
+# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes.
+
+# This enables basic gc logging to the .out file.
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+
+# This enables basic gc logging to its own file.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>"
+
+# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M"
+
+# Uncomment one of the below three options to enable java garbage collection logging for the client processes.
+
+# This enables basic gc logging to the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
+
+# This enables basic gc logging to its own file.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>"
+
+# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+.
+# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M"
+
+# Uncomment below if you intend to use the EXPERIMENTAL off heap cache.
+# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize="
+# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value.
+
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run.  $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# File naming hosts on which backup HMaster will run.  $HBASE_HOME/conf/backup-masters by default.
+# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters
+
+# Extra ssh options.  Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored.  $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers 
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes.  See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands.  Unset by default.  This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false

+ 196 - 0
hbase/hbase-site.xml

@@ -0,0 +1,196 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>hdfs://nowledgedata-n7:9000/hbase</value> 
+  </property>
+  
+  <property>
+    <name>hbase.regionserver.dns.nameserver</name>
+    <value>nowledgedata-n7</value>
+  </property>
+
+  <!--
+  <property>
+    <name>hbase.regionserver.info.bindAddress</name>
+    <value>192.168.1.35</value>
+  </property>
+  -->
+  
+  <property>
+    <name>hbase.tmp.dir</name>
+    <value>/disk2/data/hbase/tmp</value>
+  </property>  
+
+  <property>
+    <name>hbase.cluster.distributed</name>
+    <value>true</value>
+  </property>
+
+  <property>  
+    <name>hbase.master</name>  
+    <value>nowledgedata-n7:60000</value>   
+  </property>
+    
+  <property>  
+    <name>hbase.zookeeper.quorum</name>  
+    <value>nowledgedata-n7</value> 
+  </property> 
+    
+  <property>
+    <name>hbase.zookeeper.property.clientPort</name>
+    <value>2181</value>
+  </property>
+
+  <property>
+    <name>hbase.coprocessor.region.classes</name>
+    <value>org.apache.hadoop.hbase.coprocessor.AggregateImplementation,com.sdyc.ndschedule.hbase.coprocessor.RowCountCoprocessor</value>
+  </property>
+
+<!-- 
+  <property>
+    <name>hbase.coprocessor.master.classes</name>
+    <value>coprocessor.MasterObserverExample</value>
+  </property>
+  <property>
+    <name>hbase.coprocessor.wal.classes</name>
+    <value>coprocessor.WALObserverExample, bar.foo.MyWALObserver</value>
+  </property>
+  -->
+  
+  
+  <property>
+    <name>zookeeper.session.timeout</name>
+    <value>180000</value>
+    <description>ZooKeeper Session的超时时间,调低该值可以让ZooKeeper更快的发现RegionServer的掉线。 默认为180000.</description>
+  </property>
+  
+  <property>
+    <name>hbase.master.maxclockskew</name>
+    <value>180000</value>
+    <description>节点机的时间和master的时间差距大于30000ms,就是30秒时无法启动服务。修改各结点时间,使其误差在30s内</description>
+  </property>
+  
+  <property>
+    <name>hbase.master.meta.thread.rescanfrequency</name>
+    <value>10000</value>
+    <description>How long the HMaster sleeps (in milliseconds) between scans of the root and meta tables.</description>
+  </property>
+  
+  <property>
+    <name>hbase.server.thread.wakefrequency</name>
+    <value>10000</value>
+    <description>Time to sleep in between searches for work (in milliseconds).Used as sleep interval by service threads such as META scanner and log roller.
+    </description>
+  </property>
+
+  <!--
+      下面的是一些优化的配置 
+  -->
+  <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value>10</value>
+    <description>RegionServer控制RPC程序的线程数。如果RegionServer内存较大,可适量调高该值。 默认为10.</description>
+  </property>
+  
+  <property>
+    <name>hbase.hregion.majorcompaction</name>
+    <value>86400000</value>
+    <description>一个区域中所有主合并之间的间隔。当设置为0时禁用自动的主合并。主合并会消耗大量IO,重负载的HBase应该禁止自动合并。默认为86400000毫秒, 即一天时间一次.
+    </description>
+  </property>
+  
+  <property>
+    <name>hbase.hregion.memstore.flush.size</name>
+    <value>134217728</value>
+    <description>写入数据时,内存到达该值后一次性写入磁盘。 默认为128M.</description>
+  </property>
+
+  <property>
+    <name>hbase.hregion.max.filesize</name>
+    <value>1258291200</value>
+    <description>HRegion上每个分区的大小。如果无线的增大该值,会导致Region停止分割。这个也是关闭自动分割的办法。 默认为1G.</description>
+  </property>
+  
+  <property>
+    <name>hbase.hregion.memstore.block.multiplier</name>
+    <value>2</value>
+    <description>某区域的MemStore的大小到达一定阈值时, HBase会对更新阻塞。该值为hbase.hregion.memstore.flush.size × hbase.hregion.memstore.block.multiplier,也就是默认在256M会发生阻塞,在写密集情况下可以提高该值。默认为2.
+    </description>
+  </property>
+  
+  <property>
+    <name>ipc.server.tcpnodelay</name>
+    <value>false</value>
+    <description>true时禁止延迟, 即关闭使用缓冲区。 默认为false.</description>
+  </property>
+
+  <property>
+    <name>ipc.client.tcpnodelay</name>
+    <value>false</value>
+    <description>true时禁止延迟, 即关闭使用缓冲区。 默认为false.</description>
+  </property>
+
+  <property>
+    <name>ipc.ping.interval</name>
+    <value>60000</value>
+    <description>ipc ping 频率. 默一分钟。</description>
+  </property>
+
+  <property>
+    <name>hfile.block.cache.size</name>
+    <value>0.25</value>
+    <description>RegionServer堆空间最大值的多少百分比分配给块缓存,默认25%</description>
+  </property>
+  
+  <property>
+    <name>hbase.client.scanner.caching</name>
+    <value>100</value>
+    <description>HBase对Scanner扫描缓存的数据行,在调用扫描类的next()方法时能读取到更多的行。默认为1</description>
+  </property>
+  
+  <property>
+    <name>hbase.regionserver.global.memstore.upperLimit</name>
+    <value>0.4</value>
+    <description>RegionServer中所有MemStore的总大小,使用超过该百分比后写操作会阻塞,并且强制写磁盘,直到占用率低于hbase.regionserver.global.memstore.lowerLimit。默认为0.4
+    </description>
+  </property>
+  
+  <property>
+    <name>hbase.regionserver.global.memstore.lowerLimit</name>
+    <value>0.35</value>
+    <description>强制写磁盘后直到MemStore占用低于该百分比后停止。默认为0.35</description>
+  </property>
+  
+  <property>
+    <name>hbase.hstore.blockingStoreFiles</name>
+    <value>7</value>
+    <description>这个storefile就是每次memstore flush造成的,flush一次就多一个storefile,所以一个HStore里面会有多个storefile(其实就是hfile)。当StoreFile超过hbase.hstore.blockingStoreFiles的定义就会发生阻塞并且写磁盘。默认为7
+    </description>
+  </property>
+  
+  
+  
+</configuration>

+ 204 - 0
nginx/nginx.conf

@@ -0,0 +1,204 @@
+worker_processes  1;  
+  
+#error_log  logs/error.log;  
+#error_log  logs/error.log  notice;  
+error_log   logs/error.log  info;  
+
+pid        logs/nginx.pid;
+
+
+events { 
+    worker_connections  2048;
+	
+	multi_accept on;
+}
+
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  logs/access.log  main;
+
+    sendfile           on;
+    tcp_nopush         on;
+    keepalive_timeout  20s;
+
+	server_names_hash_bucket_size 64; 
+	
+    gzip                  on;  #on or off
+    gzip_min_length       1k;  #min compress file length
+    gzip_buffers      4   16k; 
+    gzip_http_version     1.1; #compress protocol version
+    gzip_comp_level       2;   #compress level
+    gzip_types            text/plain application/x-javascript text/css application/xml; #default text/html
+    gzip_vary             on;
+
+    #upstream 192.168.1.105 {
+    #    server 192.168.1.105:8090 max_fails=2 fail_timeout=60s weight=2;
+    #    server 192.168.1.105:8080 max_fails=2 fail_timeout=60s weight=1;	
+    #}
+
+	server {
+        listen       80;
+        server_name  hotelping.cn www.hotelping.cn localhost m.hotelping.cn www.pogoy.cn pogoy.cn www.knowledgesocial.com.cn knowledgesocial.com.cn www.nowledgedata.com.cn nowledgedata.com.cn;
+        charset UTF-8;
+        access_log  logs/host.access.log  main;
+		include proxy.conf;
+		
+        location / {
+		    if ( $host ~ "(.*)nowledgedata.com.cn" ) {
+			    rewrite ^(.*)$ /home/$1 last;
+			}
+			if ( $host ~ "(.*)pogoy.cn" ) {
+			    rewrite ^(.*)$ /pogoy/$1 last;
+			}
+			if ( $host ~ "(.*)knowledgesocial.com.cn" ) {
+			    rewrite ^(.*)$ /sgtz/$1 last;
+			}
+            index              index.jsp index.html;
+			proxy_pass         http://127.0.0.1:8080;
+			proxy_set_header   Host             $host:80;
+			proxy_set_header   X-Real-IP        $remote_addr;
+			proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+			proxy_set_header   Via    "nginx";
+        }
+		
+		location /home {
+            index       index.html index.htm;
+			proxy_pass  http://127.0.0.1:8090;
+			proxy_set_header   Host             $host:80;
+			proxy_set_header   X-Real-IP        $remote_addr;
+			proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+			proxy_set_header Via    "nginx";
+        }
+	
+		location ~ /pogoy {
+            index              main.jsp index.html index.htm;
+			proxy_pass         http://127.0.0.1:8090;
+			proxy_set_header   Host             $host:80;
+			proxy_set_header   X-Real-IP        $remote_addr;
+			proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+			proxy_set_header   Via    "nginx";
+        }
+		location ~ /ndf {
+            index              main.jsp index.jsp index.htm;
+			proxy_pass         http://127.0.0.1:8983;
+			proxy_set_header   Host             $host:80;
+			proxy_set_header   X-Real-IP        $remote_addr;
+			proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+			proxy_set_header   Via    "nginx";
+        }
+		
+		
+		location ~ /sgtz {
+            index              index.jsp index.html index.htm;
+			proxy_pass         http://127.0.0.1:8090;
+			proxy_set_header   Host             $host:80;
+			proxy_set_header   X-Real-IP        $remote_addr;
+			proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+			proxy_set_header   Via    "nginx";
+        }
+		
+
+        error_page  404              /404.html;
+
+        # redirect server error pages to the static page /50x.html
+        #
+        error_page   500 502 503 504  /50x.html;
+        location = /50x.html {
+            root   html;
+        }
+
+        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+        #
+        #location ~ \.php$ {
+        #    root           html;
+        #    fastcgi_pass   127.0.0.1:9000;
+        #    fastcgi_index  index.php;
+        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
+        #    include        fastcgi_params;
+        #}
+
+        # deny access to .htaccess files, if Apache's document root
+        # concurs with nginx's one
+        #
+        #location ~ /\.ht {
+        #    deny  all;
+        #}
+    }
+	
+	server {
+		listen 80;
+		server_name ques.hotelping.cn;
+		charset UTF-8;
+		include proxy.conf;
+		location / {
+            index       index.jsp index.html index.htm;
+			proxy_pass  http://127.0.0.1:8080/ques/;
+			proxy_set_header   Host             $host:80;
+			proxy_set_header   X-Real-IP        $remote_addr;
+			proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+			proxy_set_header Via    "nginx";
+        }
+	}	
+
+	server {
+		listen 80;
+		server_name apps.hotelping.cn;
+		charset UTF-8;
+		include proxy.conf;
+		location / {
+            index       index.jsp index.html index.htm;
+			proxy_pass  http://127.0.0.1:8080/apps/;
+			proxy_set_header   Host             $host:80;
+			proxy_set_header   X-Real-IP        $remote_addr;
+			proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+			proxy_set_header Via    "nginx";
+        }
+		
+		
+	}
+
+
+    # another virtual host using mix of IP-, name-, and port-based configuration
+    #
+    #server {
+    #    listen       8000;
+    #    listen       somename:8080;
+    #    server_name  somename  alias  another.alias;
+
+    #    location / {
+    #        root   html;
+    #        index  index.html index.htm;
+    #    }
+    #}
+
+
+    # HTTPS server
+    #
+    #server {
+    #    listen       443;
+    #    server_name  localhost;
+
+    #    ssl                  on;
+    #    ssl_certificate      cert.pem;
+    #    ssl_certificate_key  cert.key;
+
+    #    ssl_session_timeout  5m;
+
+    #    ssl_protocols  SSLv2 SSLv3 TLSv1;
+    #    ssl_ciphers  HIGH:!aNULL:!MD5;
+    #    ssl_prefer_server_ciphers   on;
+
+    #    location / {
+    #        root   html;
+    #        index  index.html index.htm;
+    #    }
+    #}
+
+}