python爬虫，集群是如何实现节点的发现和管理

python爬虫，集群是如何实现节点的发现和管理

Ignite集群管理——基于Zookeeper的节点发现
Ignite支持基于组播，静态IP，Zookeeper，JDBC等方式发现节点，本文主要介绍基于Zookeeper的节点发现。
环境准备，两台笔记本电脑A，B。A笔记本上使用VMware虚拟机安装了Ubuntu系统C。
1、 C安装Zookeeper
由于主要测试Ignite，这里仅仅简单安装一个zookeeper节点，下载zookeeper解压后，直接执行zookeeper目录下的bin/zkServer.sh start命令则成功启动zookeeper。
查看Ubuntu系统C的IP地址为192.168.1.104，zookeeper默认端口为12181。

注意：这里VMware虚拟机的网络适配器一定要选择桥接模式，否则A机器无法访问虚拟机。

2、 A系统运行Ignite节点1；
代码中加粗的部分便是Ignite注册代码，十分简单。

package com.coshaho.learn.ignite.cluster;import org.apache.ignite.Ignite;import org.apache.ignite.IgniteCache;import org.apache.ignite.Ignition;import org.apache.ignite.cache.CacheMode;import org.apache.ignite.configuration.CacheConfiguration;import org.apache.ignite.configuration.IgniteConfiguration;import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.TcpDiscoveryZookeeperIpFinder;/** * * IgniteCluster01.java Create on 2017年6月3日 下午10:52:38 * * 类功能说明: 基于zookeeper集群 * * Copyright: Copyright(c) 2013 * Company: COSHAHO * @Version 1.0 * @Author coshaho */public class IgniteCluster01 { public static void main(String[] args) { TcpDiscoverySpi spi = new TcpDiscoverySpi(); TcpDiscoveryZookeeperIpFinder ipFinder = new TcpDiscoveryZookeeperIpFinder(); // Specify ZooKeeper connection string. ipFinder.setZkConnectionString("192.168.1.104:12181"); spi.setIpFinder(ipFinder); IgniteConfiguration cfg = new IgniteConfiguration(); // Override default discovery SPI. cfg.setDiscoverySpi(spi); // Start Ignite node. Ignite ignite =Ignition.start(cfg);System.out.println("IgniteCluster1 start OK."); CacheConfiguration<Integer, String> cacheCfg = new CacheConfiguration<Integer, String>(); cacheCfg.setBackups(1); cacheCfg.setCacheMode(CacheMode.PARTITIONED); cacheCfg.setName("myCache"); IgniteCache<Integer, String> cache = ignite.getOrCreateCache(cacheCfg); cache.put(1, "ignite1"); System.out.println(cache.get(1)); System.out.println(cache.get(2)); }}

3、 B系统运行Ignite节点2；

package com.coshaho.learn.ignite.cluster;import org.apache.ignite.Ignite;import org.apache.ignite.IgniteCache;import org.apache.ignite.Ignition;import org.apache.ignite.cache.CacheMode;import org.apache.ignite.configuration.CacheConfiguration;import org.apache.ignite.configuration.IgniteConfiguration;import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.TcpDiscoveryZookeeperIpFinder;public class IgniteCluster02 { public static void main(String[] args){TcpDiscoverySpi spi = new TcpDiscoverySpi();TcpDiscoveryZookeeperIpFinder ipFinder = new TcpDiscoveryZookeeperIpFinder(); // Specify ZooKeeper connection string.ipFinder.setZkConnectionString("192.168.1.104:12181");spi.setIpFinder(ipFinder);IgniteConfiguration cfg = new IgniteConfiguration(); // Override default discovery SPI. cfg.setDiscoverySpi(spi); // Start Ignite node.Ignite ignite =Ignition.start(cfg);System.out.println("IgniteCluster2 start OK.");CacheConfiguration<Integer, String> cacheCfg = new CacheConfiguration<Integer, String>();cacheCfg.setBackups(1);cacheCfg.setCacheMode(CacheMode.PARTITIONED);cacheCfg.setName("myCache");IgniteCache<Integer, String> cache = ignite.getOrCreateCache(cacheCfg);cache.put(2, "ignite2");System.out.println(cache.get(1));System.out.println(cache.get(2));}}

可以看到，Ignite节点2可以成功访问到Ignite节点1存入缓存的数据。
2017-12-29